demo/tutorials/llm_notebooks/dataset-notebooks/Bigbench_dataset.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-euMnuisAIDX"
      },
      "source": [
        "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUgAAABcCAYAAAAMJCwKAAAgAElEQVR4nOy9f5gcZ3Xn+znnra5pjcfKZCyNfqDIQgghZMdxZMfGxpbbwhjM2g4h2Ak/Nol3Aw5xEsLu5eHh8vCofNl9uFluLhiwhUi4zib3ZomcZBMgARsjt4RxbGIritcSsiyE0GpleSQLMYxHPd1V59w/qnq6Z6ZnNJJG/Ej6+zw9PW911fueeqvq1Pn9CucASZJokkzZaudirC666KKLcwWZ+y4TveyWJeW4/lKZYYD5mI2m8+YdH61Wk3Tux+uiiy66ODeYYwaZaKUysNSI7xSVtfj4MCPi9t8WLhzY+sADt9fndswuuuiii3ODaO66ShQSM7lvvYj8B6A8/pMIiM4/evToTuDI3I3ZRRdddHHuMIcMMocgC9ysFwx3DBzVyFzCQBpF8VyP10UXXXRxrjDnDBJygdFyl4wiTS3egJPnYrguuuiii3MCPRedem57NHBk3A6pwLxzMVwXXXTRxTnBnEmQSZJ/xP2gaDjhrv00vTSigB12tVqSJNrcf/p+uiFBXXTRxY8ec+7Fvuqq+f1RT/ktgl40PogwbKn/XQgv7KhUsJwBJjNIr10G2UUXXfzocU7iICsV9AfnL4k5nG85//zYKpXv1pMksStv+uT8eKy0RtyWqU9U8U1cU5e9Mb17qtU7anNPWxdddNHF7HEOGOTUTJpKBa1UsC271kYLjh79zyL6bnefP3F4b5JzxLEPvrhw4Z/v7sZMdtFFFz9CnBMGORW5On1V5YLVsUT/CNJrlnXcUzXg+JfU7c5K5ehQ1x7ZRRdd/KhwTsJ8JqMpTW7dzlJc+swykBZ3HpcdAfcMkVAGLVerKHl8UBdddNHFDx3nJMxn2sHMFYrEmrbtPyQxtosuuujitPBDlSDXbwgqDo4grUTtCRJkF1100cWPC+aIQc4uZMdMLAhtzDH/lo7KdhdddNHFjxZzwCATXbuWCNZO8/sWBgdfUvhuCh75hN8mM8P2djfKp4suuvjR4iwYZKLXvq7/YrGeD7jbIBxF3NskyZZ/JTc9LkyBBdP5XNxBwETV8OwwcKJSwarVM6ewiy666OJscEb6bJIkWq0uXOkS/ptqaZ1ZSqsoxQxwU/f28J7Jxzil6LwnG/aDD2zf+rtbz4S2Lrrooou5whlLkCa+LmjP8ix9KXUkEloWxBm+TaTwnDsmok+L6iHcIxcxaBzP0h98bnvlxe1szetLnu0JdtFFF12cKc6YQbprjLgiolKECzXlwVN9Fz2kmdumyPyhNLhGmRhEI9XqnceongFzLIpg0A0s76KLLuYILQaZJAobIZFZMphsgnQ4W7g7ICaAqp2oXHfs4K5dREePthsnZ2BySdPOWS2+K5bTvLG5rcsgu+iiizlBziCTRyIWDpY5ursO5PnPic8QunM3ofgvZ46T2eSp2tB04iRJYkmSpDOmFCau44x77e6II3GZ0s+U0bEyvq+PTc/2Ic8tw5fGJL5l9ky+iy666GJ65AxyydJVuN7OYh/lM88OIQwjz42QygjKMJ6OYlajhzqhd5Q7qFPJO/Ai7Lv5fx7VOHO7CfdZZPJsPtwLe9fxmb2D4H286IuJWYTqAvS8BbgsRmwAGCTL9gFb5mhuuuiii3/lyBlkqsuZN+8OsvogIaqhOgqhRikbJUtHca2TpaM0pE5afzBJNn5m/bb7VGkP8p74/3TtcSapBhODIjvDvj9I+fy7kbCGtF7GrBfPYtwUc8vXd3AIEdC5AEYXXXTRxZkgZ5Alt9yg6BH1sX5gfsHbNOdnriBQ7jVOvpRWqH72rHVYY3bGSytFNBqLkXSQrFFInN70hBffbmiYZYdddNFFF7NDIUECJcgZjytNxtiEA7iRpYqQTu2mubPMsi2AIGKz5LMCmOKmHeMtu3yxiy66OAeI2v6eIthbirVlRGGyq3imlMHJ7bbM60ICzMuatSrsTlmXRrFZqeNddNFFF3OIXEXtIBNOz5CauvfZQ0TqANXqRH47qyK5XYbZRRddnGNMlCDbMUWY7MyR2r3Ys4XjiKC4r61UPnMQsrJpi0lm+olDpfTE4Wo16cS6p6Gviy666GJuMZE1+mTD4/RcyFWsGcRzOpCWAKogHzGyjwATdPbg8QF06d2Vyv2fn75WRbc0WhdddHFuMclJAy3GM7lG4xSHSwp5QLa7W3uwT4t1easHkem1cqHVrWMi0XIXeY9Qa/LHtmOno+cnH801wydt6wa9d9HFjwgdVOxTOVya8N2W1YdE4wXi2YxH5BFERidm5u75/sVPDmAZIEsta/QC9YnHdex9GhrPHJ2YVbH9HDCsRG+6aaCvWg29k3+pVDanlcrzx//lMMr2eW2d08SVMP+lnOuPEdoz485Vptnk7LvTHSdxhbvJ04anw91nXm+hSV87XaeYl4kqdrsXe4oGOy7iWZWKVbJtu2HwfZlnG8VZPC1RCuLgbgMg/ePVfMaHLAZpfakI5gBxTOvHSUzwHGrY0zHHczXWU08tKZ8YyX4f918uwt5VwAwipfF0tbrkvUmS/EQzyZwBJkYClSo6NFRELly0FtjNll1Q1P+05vz/JJ9vF2eARGxqrYV2VIqaC8nE9ONT9lvUmWj2u2VXG9/bDbuHLO+bKf1Ob4OcUqpxIiOrVLAk+e2HIdl62WVLykuXTkfd8wCcGB78UAjRfzCrRyAzVBGapTR4jpjjbbdtiavVY+sybIUIRhaADIJHiB4DHprrMYeGxqK4HF6uIbrYLVMpXgiRBixr1EulenzKTn5skWilglarS/qvrty7LFTlNSby6gWLfJkg/Rw7rrB4FOG4kR1av97/6aGq7CXWw5VKcnxGR10Xs8Omb61A9l0OGXhQPv2tnfzOq/fOWf/JIxFLll2CPbsq3yCK6yj3f2c7d7z8xCmP37Ir5lhpGZEuxp5dCroAedl8JJQR78ElxTmJ7x0G389nnjuI7B0i8eP5+DMwysSVnzown/i5FaitI7rwSk74UpA+xFPcj7P0woPw3C42P/c0YfcBEj/R7HN6RuU+KS6yybgKKRVyzpwk9tRTjD711LQUKsC111nqba6Yyd7vZnvWPvEp9J09KpUkOjR8qC/WeXeKh7fnGToOLghR5GZPcg4Y5Lx5wTL31C2z3BSRM0jLR09H53rAHwKaUmC1urA3w25Q4ZYS4Ro3WyUiKqJ4YcMW0DyyIeBqtZLqARq+AwY/BTz+Iz2Rn2Q0JSd/7mpCuAejTKlkYB8C5oZBJolywZJBotIHSeVW8BSIEB2hkd4BfKHJJzof78rRby9nXvmjZI31CPNxi0GLpBAthCEDF0PCMCE6hNsOFu39Mg39exIfmZZJLn52HRq/DS29kbSxGhFFFEQUHBzDHUxSotJBTP+SZbs/1mSSE+MgRVpSZJP5TG5PqEp2ahWoZVcquivY38QCFq32KVleJ/rm0ATZM3aeQkCQCCd2J3aIEVVkJsn37CCtOyEPgZrgiPrJxBe/uKScuX44aM/HwX8NfBU47hlmDSyr5x+r45ZinoEQ46zGeKuJLYcfrsnjXxaaaqUoqhEiMVEMOoPD9ExQ0lVIuJjcfFYGIkLUj+hNwKn5hKS9qCwDGaD5rIWIfBGWDDzL81OiHiWEftzW4PZOeno/TmQbedm+pR2rj21+9hqi8iZEfhv31WgUIZr32RiDtFgJQRVEIpxVGOsIvdOo2DBVahxvnzkXShL42rai+0nGw9MNE+pM31w7aQzM8WbON27F2+aHgJ9873zTrnre+endIfT8dpaNxTiKoHnWapvtuWi3NRRxQ+WAethd9Ne1RZ4NJrAOn7uKqYkra3dHHLN1pPXlxeJTxRgZmN/A//vcfN75yuHpO7kb5J2FFJfm6cRwgKzxNwj/E6eGiaLWh6SvxFmPllbgBo2xBcQ9v0Wj3s/CAx8i8aFxO+aSfZcS9XycrL4OMyOUFLLDGF/CfRduI0BMlr4c90twW8d5fQsYPvY1vvuq4dxZNNmL3ZTOxnmYTGqfBQwIs+lqMmMYyw+cvEs7fXMNV/WiMlBLqJbTZ+b/SrFlF9HCkfR3Qii/O01PxiIStU+d5Kq1tiWdGoKKY/nLCEXYWS8xVKkkUdcOORdwxl/ycyk/vhAW0Ft+HZmVUVXS9CuUoktxHyREqxitryfxvwdmthU26z3kmtROTD7KC684NuWY+7/TT73+a2j0XsxXkDViSvHtZNn/4MIDnyHxlEXfHsDlA5hdipmhoY5nW8jC3bzn5QemjJ24sujAcn7w4luw7AtTnTQT4iCZJtJnbpjDqXtpqdo5q+yZ0OrYyU+usNUBk+M8f7JQLOi2lhDdlqVjfcJEdU5EUxE9CLbHPT3miKlIHxIGUF2M23KgTJb+c2znDXdXtpwrTHSyzgkSMe57bjlZdmmxxRC/n6h0F5ktQAOkfhNUv0Jy/Wm85DwizSKuQ0naH+674bsrhlny/B+TvZQSlT5CI+1HrZcQ3sBIbQtUh5CfWUccX06jDhqBsJVG9hGGXnFw2kLgL6w4SCL/9+TNp1Gs4sxQVAxXhe+rBMuQIrB8qoMGwAUTFBEZcer5pJ6qNNo5oHvSALPeczycZdK24vuslZvJ/Z+q79kEn7diECfHJZ4+vdUqmrpfEcxX57p06zeRAOJfERu7B0r76uXGcM+YGMRlPOuzLBuUwKVo6UqX8Pj1679bb94/pzqHs6F5ch/5N0yOx5yu/5lspDPRM/m4TmOeaozZn2+bdjgXKnYzHCYK1yC6ODdLZUOkPEpmr8eya8hSRaPXMPiy5SR+4LTjIrdhU45JNirPL6mx8MBfo+k7CKXX5GdkawjxAi5ccZyxxsWk9aW4QVwe4eTI3zH0qoP58dPQMA3j7BzmM9lDfJYe4yRJ7NprP/Gwp/V3hKh86cyKtqu51zJPv9DosSPAYO5JnkRnRw/73KEps+aUztx/O5NKinbTNzXl+5QPcbOo8ERUq2iSJIz3P8n5Nf3DO3176kOXKLPstxOSJNEvPzHQW66Fi9ysb9zmSG6gcLNhj/QDgeN7Ad5wVf6oVquMAMe2b0/23XbbliePHv3eFqE80hw3/y5oSzoO3U7EeJhFqyrU7BaBa55ra15a85Mk01/D6embpRNz/LgZmanl3uDmhsljnQpzrJWMMxq/CRUgMpxvsqh+jO/V/wcS1fAsJu5dRnbychLZf0rypqDDGlOJ5PNwdOMQS57bQ6nnNaR1cPqwrJ8fSMw8/Rncy+ApwgjoPujAbDuez0RMVLHbvdhNJjQeG3l2TOjrX//9pyuVe/+NWe0t7lZkjDTvvxZt4sFcbU9w2f7El39vhJvfNJinNLbR1ZG+uUXrwW6Xb6dWLE+SRLfsWhsNHj0yuH7Dp1bLtvCaRwivuA4WQBY/4jricOhasn/m2vt2fPnL6QFg+HSlnaEh9KuP9i+9Juu5YSty5XUbfCnmPLJN9nuWfSPL0scrleRwXhkp77dS2bQiwy/11FJVVVOxrdsye+3rP7Xz9a998UheZm7higy9/LrruQp0BdssAj3yCPbPlcq926vV3j1JktRnS2vISmURHURzb7XguIuJBpzs4Ne/dmRPMXPtqvN43xddtDtNkuRYs33ZZZt7zz+/foUZ860qputVATz69KEXLxh8ZvDobhsbmz9fe3rWbt2u16x3+XnB5rNBRrZW/cA1lU8+GNGzE5ITM9kyK5UkeuihRQPr19+76pFtevl118urcJaSe2VrW6scuZb0Wat86tFqNT5QqeT9VSr3l2H0cjMbaNJnKqbmCvcc2779vY91GqvOwou3bpPl11TMqIKuV0313oOPVe/aOXX/+8uZ1i6Rbb6Y9cWEVc2iikZZ+OTer3/t93af+so0X/fMnQ3yvj2X4H4NaUMRMdz/jtsvqrP52R2E6ABuq0nTAcRfxyef+wrHV00fjnMmj7Fbffx/kTpRGOWkKm5Riy+IgkzJUJstpqYaTpYUJ4f7nAWq1buOAPedar9WDF2HHzvSdy6NkNImQU50FiVJol/9av+yhfHRm116flHcLgcGkOZNEEAEcVdcUonCgbLKX1+74dN/Ua0e250kSZ0OaB9RALFQvmBwwVvUone523rRkN/iWkjiwm9GpWg7LL4HfusrkEuYW7dlG5Tojzx4DUHVzUTiUW003l+tLvxLM26UEL1PsHUQehGseY754pPRPhi9p1rt2wIc60DqjBhfkUhcPU9HXXbttYMXv+51Q8/kNHZUVydsmzcvW+we/YEIl6q4oYCLikd/0//9F38XLlhe6gn/HuRmcVla1CzNRxZXNfl3HvE3kl2wqVJJdnZikle94Y8HsrGxDaUe/SWMG9xYIKoTGEkeiqcaiR5w2Oos+KvLLttchXqvubwHid6q5PSpuEnQ2C3aWakkV7WPmSSJfvUbFwyW0ujDbtnNiqSIqASNStjDwE3ttFUqj0Rp2LU8ePRRd7+6SZO6mmsoq/EeYBYMsg1z5cVWuYFSOSIdM5BDYE8CUPf9SGMvImuwFOLyJdjoCrj7mbkZeCMs291PI1pNVoTqiB7ETx6j96U6dv4xJKQgkGXzwS7jwgMPkST1001TnL4e5GScczvfRJyWLekcO2m8k/yfJFqtXrA6RPGnIPrP4De4eb+54Vkzxq+BZ3XcU8AjsJUov68S3Zux4M1ffGpJOZfiOp9MMeWxpPZOJXwUZL27q2f1vN+sgWcNwMuOvxENH69U7nvNuBqdaU01KEgZJ0aIVUOs7ksz+A2Nev4Q/Grce90LWpv9muFuKyF8xCj/1k03fXL+bOIR43qtbm7H3a3wSkPLbCD9ov7Rr1YHr9iya+2kJYc7I4rE0JCiGmHEOLEEjZQwX+q22qV0r4j+O5ylbpm25iWPrQTvF5O3u0QfzbKB1ZP7r1TuXRzX7UMq0cfBf9VhgWOYNcav43if7ubmy8F/TSW+5/zz7feGFv70sKg+JSKG5/RhRSygyKpG44LBibdNYpr5MlFdKSqtawORO5dWKpsXTKRvm6mzGMIyEYnHx4AyeE1cpkioM6KIvT4rJIly/3f6gdcXy6AoIjtI64dJXHnx+SHcniCKR4EU95WIrJ05x7oN0wljSaLjtsK0VKHUs5YsNZAU9ypmx3j+sjruu4ii44hAWu8lKr2Z2tjVrL0tym2ns4+rzXecHObzI8aPX9zb1HmpVC9YnRE2icrNbul890wR0yYrLbJFtJ25upu6W+yZXy4e/vC8kcbNUyWacS++uhuOrBb0P7r7cstSLVxammcESB5bKK7uZu7Zmgzf+NBDixbkc+i1PI7eQUxx1KwRu8htKuH95o1lZinuZjjmbX2Cq3umjs8XLb3rByd1PcwmaPv7I0L2zyI6MjHeFXAzRG6MNHzugqGhjZXKp9aQd2rkJocpfTcaYybjBUscxNUtU7N0tbr/IcgVbhYVvNha8yKKgONq1oiRaL2WSu+f2HuirtHHReTd7tni/HwzBVcBXFAR1bbzUMSa46+QEH9w4dDQ73iWPSOqRxAMseJ6ZIjo/FJJV7aGK87RwnJ3W+qeX5e2/QfNGmsLm2lrPlJdhtsCt2J/DNEA5nvghT0zX49JmCsnTb1+MaXyGiw1oEaWfoOFHM+LSVyfYjwOHMctIksHiEpXMbCvb+blpAtMJ4s1+cLi564h6vkAWTqAqqL6NHbyAY4+MAoYFu3A/BmcCDMQ1hJKH+NY/MbChpnHSs6Clok7zCgl/ngwz444x8JtK+snI0kSrVQ2rXDCx1R0vecXILeL5a/nVELphIjsNfc9IcRDImEiE/RMRWWxEG2+9nX3XXLyZKaTw2HGz0noBe/L/1VUo1SQnKG17SqCmmdpFHpeE+L0LUmSqKnXJ3QoqHtWBrnULFuGmZL3aaKKeMs+JCKIiLplkWe2LEjpjmp14eBkp087kiSxSgUT9+2CPi46yd6UF0lWz7I1IcT/u0v0j9dtuO/Prq3c9+bXfnXJsi1b1kaTmWSppOZNHWe80ImD+EoRvcIsNQRVVUSDFT/bhIQrcfWsHrn7r61ff+/VkOhll23uXV8Z/AOV8KtZNtYLFo2fN2IaolGVsB9nt4TosGioC0W/goJFWVbrDaXeD6Csc2cvIupe3C3uphppBs0QGBLy1Etcf8GzbAGeL4ZXVLMy1aAeqOQ25MSqVbRaXdiL+s+6Zf15VpxAca+4yN9Xq0n6Q800ShKF65RM14MMgqRE8X5UHmf32nSciVn9ScZGnyaKQQKIVuixaSs2FCgW4ZMyJZayaPEyNn1rBfftXcnmZ9fw2b03sOQ7mwjRf8fSy9EIgj6O1d/LnWt35IxPjLtW7SPLPkb5vL2okku5cimBv+Wz+/8rn917Awt3D0JVT8UoO8dBdsT0XChx1yLwfE6QnKtyTKeBiT5yz62CrrlDRl+8WQjXFA/nuKoooiaqO71R36QavknGaCb1derhXaJhvVsWk8cwqVlmqqV+Se0DIZTeZ3gqjk728I8nZmrY75buMOe4qi4vJKeBPPOkuZdHZo35SrjuoccW/XUkmRVse1IuRe52EpW6oI+aNQ4gUtYQXeKWXTJZzc+7tyvAlkFy5NRe4Rf3Zb7gc0HjNe4sds90vB6ooI5hWcMQ6ROJ3i6kb45i/+bCRcf/qlod+AJwqOmpbzTESrGk3kZ38yxwN5HIVGSve7bTzU5I0NWIrMOy/lawQ26nVonVqN8CyWPnnffpimjp7WluP8sZjjuCGnAo8+xz5tnfSxSOq9sKcf6tiLzV3fpaHmGP0sbYAkF/CU+HNET1jCxu7w+4qDlfCfDahs0v9ZTWuhvuaZt06nlMs8vP33LL5t4vfvH5WrWKXX2j9pbSsAo3xX2cRvdsGPWvz3wXT4OzYqcb4WX7FuPhKtJ6nKuxjd00xiZ6qe+6aIRNzz6I6M1kYyC6CgmXksie6SvxCGCgcjla2gyhmTgQgffhtpigfWQpwGG88RUyPs6RVROl6MSVIzzEon0fpjzvD2iMrSgkXSPSd5Lpmyj1PsqSpV9G9lQ5fGR/EfIwTbmzM1GxN26EJOETu04ul2dH3+S/IhHuhoQzn37PDAKf+NWxR39/Tc/TZ9zPHKAV4tPGpAQbPHpk0CX+JfD5tN9qriYiJ9wb/3HDhmOPNjfv2rX20JEXXzyo5veAXOHuxUPratYwDfE1sTQuMbfc09tWetidIutEdpqnH80auj2ObbQRxgaiLHqnavR+t6y/RbXg5mgUrQhZulhdzCfFIgKIYwh1N/usRX5P5DIE9ahhsiYS+SOQi/OiGQV7dVPQxYJeDDyZJFPDh5oowmSoVuVLnjUGRMNHRaI+LyQ9mhlJuRqf21CFPjeviMrlaPn69Rs+/alq9dhjlQo0GuDixaJtE9ITTTQC829CfaNQ3yk6r4bbYkPuFA3vxrK+1jUS3DMQW1epbF7gkv0i7oMTcyDERMOwe/qpejn77BNfPj5S/HCgUhnYax56VUu3uzVyVb4ZDKa6yiwbVbeaIHFz3twzcF9dqfzU/GolGSZJrFTZNGDua5quxXH2KCi5mr36e99rLAP2QWKa3dcHvpKiDB5Cs97CHjLfe0axn2cjfiRibPrWKuKe1aR1I4pr1Eef4OjQMZKLWiXDAHTvw2SNEZBeNJSx7A3A508dD6n9aLSu+D9/EIpsXxr1lHweTiD+jwhD42M2+22mG76w6i9Z8u06qncRxVcDZRpjIKEfsVuReAORfpNFS/8W+/W/hOTI5MIas3fStIjPaSharqzE5f0CH0T0g4h/UNo+p9NG9QOi9gF3W3c6FJ17FGxSvJYSLnbzy3MnRpukpaqI/7Xasceq1evG4yIvumh3uviCC3YiPCAhGqG4PXMV1k1hIHO7HogmhDMB4KYhOu6SbQr0fimOXzherRwd/cbDJw6JN+7DssdEI9zb46QwdwZClg20r/Mz3qNDblPXrZbJPVE2dLBaPToK3x95fWXom5h/yt1TL9TUNptqZMgrZjNbuap9dHRkJPoTJ/tdYK+GWIubfeI5NhklmbpZn3t2q0rPPSkL3ghAb/uuzZNonoupB7sbjldh5ESlcnQUjh5Q5L+CPENbFXvH86ElLDUdW6caX+JmOm4eaaq41tiRxvqnN13ZZI5JEat5/DCBexxLc2bbJMrVzfpBBtzTWq5mA1DYFcNSiBZX8pU71Sxbi2XL3QxcwN3cyRMn3Ey1NKAlXdOkO8p8qbstd2tZs91NPfUdUDsx1ck3C5ypCJO4cv93yki4nLS+vAinOU4WHodKEaeZaDOPmedX78PZQVTKGZzZhsK5MzM8HSUdO0ha309aP0BaP0jWOIGIUe6NCAFCWM28+R/B5HMsfnbdxFqStOIan/+fX6KR3oll7ydLdxL1KFFJMQNPe0nTDcTzPkKJTWzad3F+bMtkMdFJMytPdfHMFXMgSorIqED+cUZo+0xoU7RpfSb9PuowKh3X3v7hYrKKXbzv64peJyrz80IWkjNJF3PLhh17II+N22btQc4PPLA7bbhvxX1IhOYDhLtoljV6Bb8cvJ/2cnCOiahmWX3Ig26tVr9br1aTwsaTWLX6vhMmfFk1dApk70uRPjWxKdIjmCg1cftiFA0drFQo+kvSJEksy6wqovtVWyFN7m6ImogOMkskSWK33PJ8bfsjd/1pGuQNZul/EtHdGnpG8WAgaev9InnxCnE1y2K37OJI40/Bomva+2wG0DuF9CiyY/vWux6qVpO0SX+lgp1/vu53T3eIaJ2mKNw80r2XNLrW8pTGCVCNMOVvH3voPUNF8HdxbP7/9q13PYbzpIQSTAjeFVWVsjsHRQPgzegzk1CanyKrxvcN4ToJIXYc1Qjwb6roweZS9OY+X+DSSmWccV+C+4LcOQOCpqLhmEn29Wrl+8OTVwSdHs2XPGcnQY6MDRDF16MaUeqBsZM7iE7sbDk/ig9AIinIA2SZkaVQ6lnOWHrD9J27FXRuh3Ataf3nSMd+lpPRzxHkZ2nUr4lUAr8AACAASURBVOXkS/8HIjuAlNEf9FMq3Uyp9//js/tvnVJkNxEjuT5l6JUHOLzyM8ThtaT1X6Y+9nlK8UE0GGZG/eR8gt5KpA+y6G2Xw8ZxJjnNu8QnqduT2y2IuYGnhtfBUnJ5tPPH2769rQ0pWNGWVPxUl3ASPefAf9SxSyNCfDWiJmBN+5yoIqqHTfwAdPbC+1jPQbf0cBFnaOMrO4orooOO9I+rn+MQBEZcs1pnlVYONetHTiyI45GgEaRtFq6m1wIDHcnwY3n17ok9RlGoC+SFSGWCGwiE0yrc25yHbzx858Ht1aGN4v4rno19VFQeEo0Oi2hK4RgaL3snglmmDstd+DCjcVSYGZjw2hJBjCPFSBPu48sue76myAtISPPzLc5B8nMQZRVu88enq/g2S8F9GtNOPoaITPrdEcFAyiqyF3dEirAmwRR6BVlRrWJr1xLltlyMgkE6uh2V/VLEznrWKLv5RbCkH8Al/KxoZDhWOHNURA+QsTe/dKeTauhn96wkYvREK/BsXe5gQlGG8f71fGbPGyd8Fu99I5959k14I8ZtBFFDxBC/iS27TnEfSUqqdY6uHeWui0Z438tP8K5XHuLoXzzO0OGP4GPvIEv/BNE6acOwdDUiG1my7JKOITxNafKOl9c48ud/g/a9i3r9DtLGnxLFJ9AI6jXQsJhS+WMs3bOqGZI0UcX2JuMZt8xPbY+jzSvj1BCpC1ITpCZyZh+EGlBDfHoJshN959SLPSFPPHZncOJdVgwucjzKQsfAb0isp+fQMHBMVWkvC+wO4tILEkNhMyzGbf2djjKvNfdoUz+104RMYbyGTX64kiTRRqTmkp9H03c/V2+gavWF3SLH/ou4v8fTsd8F+WNURmj6porxRFDPUhC9JoR0DWitKfw0YwUACFNfpM30wsyzurTJSs1XiLur4QvcPPY2ppFL9lkaEXUMiG97kRwZZw5FzwV6Ef8ndxsZZ+aOmmW94K+47JYl5YGBwWU4a1pFkQ1RnkD0ADC+sJ1GpeVZyJYmSaK4r83PurjOKlia7g2hdPA0pr5F55nGQTbVV/cKyCCWKY0xQ/RWouiPCD2fm/iJ/yj/lN6PWx9uSqMGGl/B96KVM4fYOJTHtPOyC9uMw2v2kcUfAdtCFEd5LCSXIvqOZsjYVPrb7J53Lh3lhVXbKcfvx+obCeEQGnImKXI5pu/gwgMxietEFRumMsJTqN2ipDmDo+ZCzdXqLlZ3L75ltm3qAjXwus2kBHSi7xxGII0/jrnEGkkeqNuyXTVvXJd6o6EdCysAVKuYIB0YqBgaVCZyiVlh5uq92Sn3mA06BsmfEZqmgSStVF44uGHDi19qjI1+yN3vEuFA4T0eH89xVKLY1K91UqWI5/TCwTPZMz89/cW3FDpsXso8br2AJrhL0jRk07zkmpCxcRW6SamBO+UU9uCyVzQycTcH3LNYkRXn/yCdLxGXiJb6MENENEsbdXWextLv5jZJDMHcWCoNX/zEE6v6EFbiha3U3VTDCGL/dGYLuZ3FszLOYPQNSGFL1qBEpQFgGSJLO390MSGKgNzuV4oW4375zI4agU5l9NvV96MrhsjsHiwbHY+Qc7uVe3f1zZgt01L/jRUHRvDz/gRr3IOEEUQhrZcpla9mNFsGc/AEpSmIWj2gGJh625uh+aKcZdudVHBcT9MGOUfPcLWKVSpphER9orlHeFzykkLddclVhZz28ZqGDr2lkk3jUUy0Urkwdk72NVlqy/nh6m41F6nLhBqJZ4hxlTLMvN8s0KJzbkX05hxVKsnw0MJlWwaODcVBo4+5Wb9IW9FVHHHWgMduTRUcaIsBPRXG59llvOakC3VEwFrsMZckJY4yZszbdbfzRbStXsr4CGnJ5TBBtnor9lFxjBAPYukCsNeqKJm4iUQK2d5K5ej+rdsu2Ccan3DL+t1dRWxQRFaMjIwckuCL3VtXwtyPoZxe9kzz/Jrc8UxtkPfuvRT8NWSN3K5kthfP9mAetdJrOw3tA2i4FKxMo94P0ev4+D99ie+fGMkXy/r26dHRYq5P80f7dhNK64qCFSuQsJIkyVMaT/UCuf76lOQRWPgzX6As/waXDQgpqsvRxjIS2TdRxT6ddMKNG4tDPBWRmkNNoO5IzZGaS/E5jTbqNReti4fTu4RzJEHmapSWaa7SKC0lU3Nj4xFROdQ+Ty0Hji2uYx09dEkCjdLIgIsvNjOgXfoUHDuheYXjlq3wNJhS59PPOM3whNPs/9Q4VQBztZqkg0d3W+S6WzU6RFtgeZ6P7gAxPiGb5bTombCvkJfTcx8SpD6+zEfBdTVEajbVeVOcSxF9wEpErKm+53lNggjHwWrm2T+4pXVENF9SRUxF+qGxGPe1ZllhRwSQJ5MkMXU9KKJDCCaCOl520VeGYKtVS3mWkGOiQS2r71Orn17udfPkzxYRNxKXI/KMpRouG3n+lb+Enn8bPaXpP0HuIpSeyV9KppTii+ntWwnbjLMNoHbJFwVzz71sQeaf4ohJqBiMHaFeP4Bqmj/O3otob37Krb9nhsjNTWuKmEEuR07Rfjrxu6nPjpF7XSU79xLkxLp/UKmgSZKk69dvWolk42EW446/nA8edOGo5OEhxc+Cu6mIDqpwCbBzciB1ksD6DaxRiRabp4wvN5BXuUnF0n2GRHqGrOicmmDPoP9OZdSa8zxRwk40l9qzMnh5siMwd1n5CYR+0dzHebr0tDQANHegaOruB1TCCcda0qKTB4wrVyVJ8qVOmkClcm+fua+T9vvZx42jB8BHXMMeNfYDa8wzlTy4e74RLhVhZV60Q3C31Mi+AZAGORwsPYSzGjBRAdFV7vYDFaWotI5IhEj69Wr1fSfOrIiwnNnNkiTKsn/fT+Pk68kaoAFE9yAndwDw/JJa5wML5jfwjv301J9Gw7p8jRlbidvFcN0cxDrnWWb5v2ago62c71nWg4t+2vAf1HKeZNY+SR1Y48RMjqntAm2MXyH1fGU6y4qU2BwtBaa1TSe1WxARyzNWbAYJshN9p4/JD0ClklCpJLr1Eb9LVPvNsjw+zwsmaKkiPEua7XMNI7j0uuQ5u7ntSGNxfxvwp8UImveLwoVRaiOvV2WBu1vTGC+CqZaGU8+eELefZ8JbY/bnNc0V4mwtKGf2LCVarS5a7mK3O/5MpXL/1mr1jmm88HDllQN9mcstkqYrEJ9EsIDotwS5zJuhQPlmbb+zZsbE2VEJqWm6C5FDIEvHexHUrAGU3vjwwwvur1SS/fnSxq2eTLhRJVpheXC7FhRansrOznovwyHzuro+jdvaptfZ3frEea2jA4ghqoAcDsiTAFHmQ+bZXtFSxTyFzFXUVpl5LJKNu/TMGmTIGdZXPxsv9kZo7LuEnvJqxk6ChgjsSYLlDq0Z6ywmyvFVIyx69h+Ie9/C2EvzcesnlK/ip1Z8gUsPjHB62eQth9GSvQO4ryJLc6btNkw9O3L65/eDXlwGsbQo2yajICMwOdVwfIXA5k0jrfY0T4umpRTSmqOWhzugrcfcaQmUxcbJAmZ72y0X1CSawYvdib7ZY+3aJB4cXHS1iS/1NN3nrieiKMRbt/pKUb9DVG81y3TcvuS5ucXhYObp0yX1Iy6lRxG/Ec8lcgTFUtMQ3bi+cu//1hjr+X96eg4VMWoLyyYnbw3S83bL0phchcpVJtHIspMHAjxs8PNeLHrkM7C8TpjgZsgdSLTbICevHHk6aB07OyRJYus33Ls60vPuzGxsmVntmfWVz2zH7B9V2Z8GhqJMLAvSGzJfaeLvwv1N7lY4UYq5QcnS2qiKPezwC+30nO55tJ+/4+oi+ywd+6ZoWGd56FbO7NxNlLUhkg/Coru3bHnhcJKQVqsXxnnNR/+ISRp5U5b1XMbVEO03sr+76crjI7t2ra0NHRv6Bwi34pTzQPJ0PrABsd7WlZKdwJE8E+aukfXXf/op1WjY0rQ/L4jhqwVZbtbIox60hFu2uyRHnzytk++E5vM203KsTSSee5Nl6XqcBagaGp2g0djG80PD8MDMYyWJkWxULNpO/eRhRPoRNczWMy9dyrZte1j0zkkHzeKhXvJ8GdffptSzgEbNiGIwHuPFVUdy73el5c2eaclZqkr2skvp6bmYRj1Pa/TsAMYhEtepSy6cUT1IrUsza2Py8ZM16RnahhgK0YTg3kk4i3qQuXTzU72m4VfE7TcJ0Ql1GTUhQhlAQtkss0lDGGAisr3k8QGIR8xH/0IlrMN1QdOp4DmTBJcPx3Hj1akt3HbttYxmLlep6O2epUvBtWlbaxaeyCz9XP1kOtRT1gjBcLS9HuRsMZVlZMW8hDNijNB8lGdPS5IkumULkWSsymx00N0jCdGlAusMUhOGg8mwo6mYlc19UDXEmRW1KNqcHqKKW/b5RoPDUezllg9b8NNw0sCkF4N7/gIJ/ldCuFHUV7lleYiNoG5ZJITbHR+8YHDwi1+r+rGgtVWWydtEdY2bjWsADiaqdcuyh+aVSzvzEKPd6QvbFz0j6BHwFYVwoUBuG3Mxx8zddo6OlIab8/a17faMWXZCkCKHXGKYGHcqKtXqI8k06uypZ2EqNkIyUzTARqCqLBlcisZXktbLedSF7CewO2dC15/aX5CIkTxygMVLHyOetzZP99OVqFxBkuxm0+3ka08V8OKZvo4iYHsjucpaqM6Lvr0Az94KelcRagRuJzC7H6rK4LLL0W/3k922k7suOjI1pKjoKxHj3r2XEOR3SRurwYxo3ijpS9tYYIcY6iRBTodpHDgaxtLM4xqSV0M5mzx4AcMhUzk9G+RpPC31uBzHKQs89zAOoDIghSrtZHnwdrPb3GZlInoos/pfBV48AZDFi/5eG/yChNJveFYvN1W+/CR8vov8RkDfCpK6WX9epqrlnRUXE1V1S78QGPt8Z4/zGbpG5Ix9lB26On0MDv5Ur6Gvxr0XUMtSy/3FROLaj0o/4uNOmMzSybdWKqqK2ZMe/F5ixnn9mUnAHc6jAcdeHHx84cKhTaLh4+QRNCYi6oJC1gv6JhWtAKPu3gfEZqZ5EXsHxDSUEOdxs9q9Dz74nuMA1eojkbL7oIscQFg5ZXwRUwnHzPyfb7nl+RrkNuqr3pDuK9X0gGi0sjBUNZlwbj7FasC2fP8zWXvHARRLI5yL2LT3ZngO/Fe1df81K+Y3289C9DLDWIPIxUVoD2SN3YTy1NUBZ0Jyfcpn9j6IZe/GHUKIsfQm4E8mO+EQYsT72D04zIW/njK6OyJ6Wxn2LiCTdZTC67HoTbgtAIworuPp54nqW7lwRR+mb0PCrdT9m2za8yD+rd2kpUMMMMxL56WE28qk+xZz395LifRdIFdjmVEqK86TpKUt7H5FSlIwtdmZqjo/sHWLLcJriMbkthhMMHVTkyh32bppvq1gPqKFimJKsX+zPwXIZggU74RZPjdJkthrX7u5TMziwnsMnqdw5fbrdkkjV/5D6BnNvPG5gD7ctpzB0A03fOIPGo3yAo3i2y2tNyWaXDV3U3fpQ9wQz+v3FZKPoIiqmttXAvLhavX7w5XKwl6bUUL/yUA+v5+YX4rDxS5mZm0vnPwFpLl0MEntzf/Ns0tCrJ6lzxD8w4svGHzm8IkXFnQebXbocGtYCKndfvvu9IknBv7kpZPyStHwW+T1N1NBiqfBcJMyeWFammuku+dZPSGU1PG9Da+//xtfP76nybSq1W122WVLDp/Xlz4jGq5xyyLaXroI6iIHVdnfnDOAN1yVnPhadeGOoGFDXui3FWCV2yzZL954uv2Y00I+x0paLxNKt1OK3zTrl3CWlUkb/eBQikcYe+kJDi87cdqLcIlvJ02PoNFg7qxhPZv2DY4vP49ofhvI5YSwGWSYWqNOiCKM+USlBZRKg2SNATzLmWpcTmmMfYGGf5yja0+waM9yovJrEF+KyFuJz9uAZ8fRxnFG/BiM1ElLfYQwSFxaSv1kwWR7FPchxkY/xNE1+5vnNlHgG1dX2yeu2e7MhcolTOCkZz7q4qPuPiomNXcZFfOamNda2/Lf3bzmxfb8t3w/cR91l9FsxjjITvTNHqVSvdexQciZFS4mxSdPe5O0CKlINcRDDat/eNEFA/8lL4TQujGvuebEIZEjv25p/ZOi4VirTmOzVqNT2NVM0BTHVCOTEB9yz/6vQPquavU9z7Q7AYq0RcPF2p+pjkGzraMoDMtN+ovtgbT15kvHf5dgrRTCTjjJeICqF7RIUQl4Fo9DVupRkFS1NKIarIitMRFJBTWcPG3O1fJ2HjKjoZRq6DnmWf2PLbLbtq8/+vBFF+1uuw/yfvL9i3Oc1eOpNK9JM60xyyIFuPLK4yPnzcs+hGXvFaI9QeNiPClSIL2Nkef0qqppKJ2wrLElqzdu+Ub1xR2txcEAEnvqqedruD2hWjohzb5a18c8G9sD9XEJrOn1D/A1MwMN7fsX9gd/cmysMTQ5rXLWEPL7BAHL+qifXEy9NrtPkzlqgLQxhPmjpx2ek7hy56uOoeEhQpQ7Yks9g3h6I9Rb9ImmqPQTQoWo52ZKpbcQ4lsJ0QbMLqZRGwSUuHcUZD+1l95Pze7k6CtypqZaJkQpUZybIhq1ftJ0JSJXEKI3EUpvRsONWHYJjbEBRCGeN4LZwzTGfpGjax5vJ7tDPcjJjHBm8axu5BWfFdP8T4H266gdtnVoN3OwZ7JBdqLvtKSvKBL0sKiWTaQPtzJ54QkDqSMyjPsQlu0Usb94tPrbDwM8MMkWXTwQtUrl/g+kfvKL6nabhJ5LgWW49UlegFVB6yI6jNgRS9OnTep/dnxo0WO33747bYZqnH9+ZN//QXZYNX7aMFQL35UEGo2TB0qlUsfsjgaMlDXeIRN0VDFERyRNR4AR1Z4draI2CrghOuI6Ntxxek6GNJSj/aj0mQYTXB1MpaSucqjt3Dvi8eoLB6+5ZvBOVasgvFajaK0QBtyZD152L7SWfC2WuiDH3bMhz+o7UR5UOfbQhmuxR5PEEhK9+sYoVQ0HBN1pmk2gJ5NakW43MaQqSUA0OhZC/DRCLG03mkjpsPjJ0eYSq0mSjFSrfLbuCx8LJreFKGxwD0vzXG0rjpVUJIwAx9zGnvEs+++qjYe2P/q+E52X+YVqlR0i4fEQlZY1tzuYalxv1EYeqX69FarTCpy/d6e7PR6intjVinPNXyBpdvJrPT3DwzOVmpsWlg0T9T4DVj4jI5ijBUNTRr/3GPN69p7u2i7jCPwVIaxFepSe82Cs9mpMHqdU3oPQh3kZiPHm85NnF0GooTJKo3GcNN2PNZ5ArMp7Xr13Qmrh86v3snTPHWR6IyLXEc9bBT6AWR9mEZiimiLRKBKOU39pH7XRv0PCF3jPq4YmO67yJ+uze2+g1LuZdGw5WTadwp3r6I3aX/Kq//W2ZFvFkkTs4986uQLxN6vPQV5b4eixzKvvW3teHmN1775V9ER/i9uaYvW0Dge6EfVAlj3N83922UwXr1K5v5yFk6s9s+UqMmDIAnWPwVLxMOyeHVHVg8C+SuXo6GzVmZtu+uT8kZFohUS+SmCxYX3iquJ+3NWPqLf6hElMJkn0tV/tX1YqlQbaOWFQVxdGouzY/k6LTV150yfnxyO6KgstVScGsiAWsrGDJ08Gi+Ppf69W33dicp+33bYlfv740Apx+jJrHRfU1cZKx77xjTtPmQPcZBqVyr19WQjLQ9YYNNEBy7yfQF4d3RkVYVjdh0APQe+havWOGsWSuW3ZNhEsXJGpz59MTzAZrlbv2teJhqtv3DQY123p1DeLpmPn6/6nvnjnuFzelOB27VobHTl+fJVYusKdpYL3g0YOI2I+BHJo3ryePQ8++JvHTzUHt922JT569IWVmUpvO90A3jN28B8e/A8d+kj06spPrw1ZiJvX7FTXa1b4410D1MMymqnFTWGoUXzP1G7/PxJljCF+75WHzogOgHt39SHzVhIKPpPKML3hEA1bTqO+gCjqwzxGPcI9ArW8iogWoTc+hDeGOLo2v36d1PymY2fZoX7Sl1biuhjxAdA+3CPUR3E5TqZH0Jf28Z6fG5qO3JzbbNqzgZ6+zaS1FTmX7Yj8DdKo/w090duS766oJ4nYJ58bXeaZ3+yEGMfOyktjBqpIJtX3ru3J04U2P7sGjf8WfNW0DNLdKPWAZzt41yt+YeoOE9G+/nG+ZOtLOjT0Xbv9dtL2dZFP19bTYgxJBBcW8/jdZimufK3safucSXWa/phKBW0vedUsk9XcNt3veYzf6fU78zEdeimqgrevTz15/NYa3zP1e/r05BELE49p+3WasI8Wc06SRHftIjp69EJtv4ZF37Ocg6nX9NTzOPGY2V2vU5Exi3VgZoWqwjY7Y+lxCj3NcJxpajlOe9wM+0zYv2CUrf4Vqkwc8+4ZUxJzbrP52Wso9W6mMbYan4FBaqRY+ijiv8Tzq4+TiG1+1hec9Nobxa0X1bP0oBpmmhJk+/f//P88kCSJsenZKwjRF4EFZOn0EmRpHmTpdt698vrZj9fK8ICm6jIXC4ZN7vfHbRGyHxXaM2pgbub63GFittWPN61dzAKniovsACFxZelzl1Cat5n62OXj3qGOfhkB1b1kY7/MC6/eTSJ27y7vS8NL17iEQU5Zx/HUUPfR1OZVhx/gRJKIsXnv2xG9H/N4gkNmAn1uxL2QNv6ad6+8bVYBsF100UUXp0CzWMUwaTact8fTuXJMKExrRqmnHymtgbtJ3PXoEDVTjoh7TfC647Uz/Yh4aipDw0O0ORDCL6AhHndZji9X10afA5aBUtjHZrn+bhdddNHFDMgZZNw4QTZ2pChZNFHymqzSZul84Cou/PU4AZLrJY0bHBHXE47XBK1LpnWh7XPKttcFr5tRH3Pbz7a7cxru/04ZYUPhYe6cqSPFtiyFzJ6d+ynqoosu/rUiZ5CH1p7A2UUUj+YS2jRhMyJKlsbEPeupp2uboVBHh847JioH1b2mntZUqam3fU7ZDjXB63h04OSreo/AxrwOx8n6G9FwMWld8WncP05RXUSOIeSOnblcg7aLLrr4V4vWUonC0+CdY+Pa4Q5ZuhbRm1m4u5ck0eR6SV+M4wOWlo5khLq518y9ZqH4tP/f3m7bniHHYi/tTUQsgTzfslS6sxhzyuJTEyGgYTcuh7r2xy666GKu0JLKgj5NOnaIEGkH70wbXHEvA/8WDVfkbnTX5OVSmzcW71NPjyleV3wio/S2Txtz1NTrkqbH5WR939G1jJK4suSpMpK9EwmvIa3TvnznFIgYuGHZDsbsBFw3RyENXXTRxb92FG5vMf7XoSNktpWoB5gpk4XcIQIr///27ifEruoO4Pj3d869972ZvsQYnTCRYEIYUpmFRBoGXdVAd13ZVpe1QWiKWVYLUkrvUIrYLooUq6YuFARtCy5aKaWbDLRKrS66KLY0dkwlZpKZMB3j+ObNfef+jov73sub/2/GSSPl94FhOMx973Bn8eOce3/n98P5H7L/vapgZR7d6RPS/O++xrRGuaROm1LGIJIUErQQ6fsJWlR/06IUuVxvNqY/Or7vWt7dGWvjXlz2CGW7AVvkcImAS66i5RvMjy2Sn7zpLWONMf8fVi4Vf/HPu3H+LYQM7ZSFiquu7tWHFCWtKaF4lVA8ztzs1W4CZh6jOzhDPSx/spdm0mg5XHSFYxnqaaaFoknQlk+GFubGaeYiSn4ugfuVQ++fILpniXo3ZTtZVeVj1ePRCN4r4v9AaJ3hyl0fbPsAvTHGbGDtXvr5f7+C9w91muC4zXfbUcnqBWX7t8TiKW6Nf+fd8dAfpPJzMeEIyUhzLoER5marPtj5SQnXM+MnYeTBYZyfIKs/g8a7KNsbTLpq/trwAq3mE8wee2GrrHhjjNmO6+Gv+3Lj7L++giQvEXWUUjcPkFW2tuLTgJbvoPpL2vIa82OLOZOdjhAb5CT2H/85cP5OvDyE84+AHKVsb/0cMaIkCSBTEB7mw7FLtno0xuymleEvzx2HH95LO/wY5Nuods4vbkkRgbQ2S2vpjzh+Ra35JqfuWVj3HGg3kD3z/ii++Bo++zqRE8Sy0TvJM8iczjtUH+Ty2GsrvtcYY3bB2kiUR8fBfxwn3fNzQjGBbljdp09nJQmQZAqySFieBvkLTt6mHS+RyiKxdJRxP94fBb5EZILa0CHay/XqxU/cOjjG7vPPuqLlr/mweQpWbuuNMWY3rB8gc1GeO/8NstrPCMVoFSQHLNsdY7Wa9KnDewgBNFR9dKvVaB2fgnMQ2lAG3TSNZ+0EikuA+FdieYqZV3Zem84YYzax/vY3jw75wu9pffIsiEOcDlyUVsQRoyMUyvKSom065wHrIBkxQnsZlpd08ODYPd0TOw165AKqP2UmTG/jXo0xZls2Xhbm0XHLhb0Mhadx8k1Uldh5ntjrM9qp5r3huG+K6+lBdBqUDPD5vjFU5eLTbJ6y/AHt1svMjTdta22MuVE2Xr3lonx05Bqe76O8iEsCzmkv6PWauMsm41U5jL1CE4N+vvsVUq0c01qL0H6C1L3I3G8sOBpjbqitHyzm0THy7gF88jhJ7Vto2IeuetPcW+XJjRgr3iuRi8T4JKfHzu74bo0xZhu2fv6XizI3PovwJGUxSZJdxGdVWbQYtfNWmV7zrN0aRxSRquct7k20/C4Mv3xD/xvGGNNnsLfHuSgzx+bJ0rOE9hkiUyRZwCeuU0OyIn1b452Pq+CbZHRSh14gLJ1hf/t1Zg62dnSXxhizA37gK6cmI/fcqnz8wHka8+dQvQJ6lNrQHlQFYlldGGVNy4beKrFroz7bUqXwJGmLMryDxu8RWs8xO36JuRG1Z47GmP+lwQMkwNRU5H4RFh+4xmO3vcFXH/0dZXsJn9ZIa/Wqx7QH5yIinf1ylPWDo4A4xbkqenrfojZ0haL1JzT8BIk/4jvH3mbiQCA/qUxNbqf5tTHGfGYDZn+vo9eshxRnXwAAALtJREFU+8uOO0aPojIBch/p8HGkPEQobyfGYbzXNdNEdagqIk18chHVC4Tib0TewvNnTn/xam8OSwI3xtwkOw+QcD2Adc9b73+vQcYhXLyDUu9E/GHSZBTxDaJmAGhs4uICoZyB+AGlTEOcxV+7zMzrrV4fW2OMuck+W4Bcrb8Rd34u4fCRhI9Dxp7EsdC5xgfFF8rwcOA/RwK5hF4tSAuMxpjPkd0NkP16W3BYWfJssjPu/LagaIz5nPoUBSp4D1AF9yMAAAAASUVORK5CYII=)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gqj3MUP46ZXF"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/Bigbench_dataset.ipynb)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wCxsD2KDAWU2"
      },
      "source": [
        "**LangTest** is an open-source python library designed to help developers deliver safe and effective Natural Language Processing (NLP) models. Whether you are using **John Snow Labs, Hugging Face, Spacy** models or **OpenAI, Cohere, AI21, Hugging Face Inference API and Azure-OpenAI** based LLMs, it has got you covered. You can test any Named Entity Recognition (NER), Text Classification, fill-mask, Translation model using the library. We also support testing LLMS for Question-Answering, Summarization and text-generation tasks on benchmark datasets. The library supports 60+ out of the box tests. For a complete list of supported test categories, please refer to the [documentation](http://langtest.org/docs/pages/docs/test_categories).\n",
        "\n",
        "Metrics are calculated by comparing the model's extractions in the original list of sentences against the extractions carried out in the noisy list of sentences. The original annotated labels are not used at any point, we are simply comparing the model against itself in a 2 settings."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jNG1OYuQAgtW"
      },
      "source": [
        "# Getting started with LangTest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "19BPyR196ZXS"
      },
      "outputs": [],
      "source": [
        "!pip install \"langtest[openai,transformers,evaluate]\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EsEtlSiNAnSO"
      },
      "source": [
        "# Harness and Its Parameters\n",
        "\n",
        "The Harness class is a testing class for Natural Language Processing (NLP) models. It evaluates the performance of a NLP model on a given task using test data and generates a report with test results.Harness can be imported from the LangTest library in the following way."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "w2GPpdowS1C9"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\raksh\\anaconda3\\envs\\test\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        }
      ],
      "source": [
        "#Import Harness from the LangTest library\n",
        "from langtest import Harness"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7_6PF_HGA4EO"
      },
      "source": [
        "It imports the Harness class from within the module, that is designed to provide a blueprint or framework for conducting NLP testing, and that instances of the Harness class can be customized or configured for different testing scenarios or environments.\n",
        "\n",
        "Here is a list of the different parameters that can be passed to the Harness function:\n",
        "\n",
        "<br/>\n",
        "\n",
        "\n",
        "| Parameter  | Description |  \n",
        "| - | - | \n",
        "|**task**     |Task for which the model is to be evaluated (question-answering or summarization)|\n",
        "| **model**     | Specifies the model(s) to be evaluated. This parameter can be provided as either a dictionary or a list of dictionaries. Each dictionary should contain the following keys: <ul><li>model (mandatory): \tPipelineModel or path to a saved model or pretrained pipeline/model from hub.</li><li>hub (mandatory): Hub (library) to use in back-end for loading model from public models hub or from path</li></ul>|\n",
        "| **data**      | The data to be used for evaluation. A dictionary providing flexibility and options for data sources. It should include the following keys: <ul><li>data_source (mandatory): The source of the data.</li><li>subset (optional): The subset of the data.</li><li>feature_column (optional): The column containing the features.</li><li>target_column (optional): The column containing the target labels.</li><li>split (optional): The data split to be used.</li><li>source (optional): Set to 'huggingface' when loading Hugging Face dataset.</li></ul> |\n",
        "| **config**    | Configuration for the tests to be performed, specified in the form of a YAML file. |\n",
        "\n",
        "<br/>\n",
        "<br/>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pHJQHDcSA_CV"
      },
      "source": [
        "# OpenAI Model Testing For Question Answering\n",
        "\n",
        "In this section, we dive into testing of OpenAI models in Question Answering task.\n",
        "\n",
        "LangTest supports robustness tests for LLM testing for now."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "YXVcv79JTAWA"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Q1uClT2kgLB"
      },
      "source": [
        "## BigBench\n",
        "[BigBench](https://arxiv.org/abs/2206.04615)\n",
        "\n",
        "**Dataset Summary**\n",
        "\n",
        "**BigBench** The Beyond the Imitation Game Benchmark (BIG-bench) is a collaborative benchmark intended to probe large language models and extrapolate their future capabilities. The more than 200 tasks included in BIG-bench are summarized by keyword [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/keywords_to_tasks.md#summary-table), and by task name [here](https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/README.md).\n",
        "\n",
        "We added some of the subsets which are as follows:\n",
        "\n",
        "- `Abstract-narrative-understanding` - Given a narrative, choose the most related proverb\n",
        "- `DisambiguationQA` - Clarify the meaning of sentences with ambiguous pronouns\n",
        "- `DisflQA` - Pick the correct answer span from the context given the disfluent question\n",
        "- `Causal-judgment`- Measures ability to reason about cause and effect\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1WO54aEnBKK8"
      },
      "source": [
        "### Setup and Configure Harness"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NQ1KF731BW5O"
      },
      "source": [
        "\n",
        "### Robustness\n",
        " Available robustness tests for QA task are:\n",
        "* `add_context`\n",
        "* `add_contraction`\n",
        "* `add_punctuation`\n",
        "* `add_typo`\n",
        "* `add_ocr_typo`\n",
        "* `american_to_british`\n",
        "* `british_to_american`\n",
        "* `lowercase`\n",
        "* `strip_punctuation`\n",
        "* `titlecase`\n",
        "* `uppercase`\n",
        "* `number_to_word`\n",
        "* `add_abbreviation`\n",
        "* `add_speech_to_text_typo`\n",
        "* `add_slangs`\n",
        "* `dyslexia_word_swap`\n",
        "* `multiple_perturbations`\n",
        "* `adjective_synonym_swap`\n",
        "* `adjective_antonym_swap`\n",
        "* `strip_all_punctuation`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a1g-SLdPJSep"
      },
      "source": [
        "### Fairness\n",
        "Available Fairness tests for QA task are:\n",
        "\n",
        "* `max_gender_rouge1_score`\n",
        "* `max_gender_rouge2_score`\n",
        "* `max_gender_rougeL_score`\n",
        "* `max_gender_rougeLsum_score`\n",
        "* `min_gender_rouge1_score`\n",
        "* `min_gender_rouge2_score`\n",
        "* `min_gender_rougeL_score`\n",
        "* `min_gender_rougeLsum_score`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WHIWMB3XJV7x"
      },
      "source": [
        "### Accuracy\n",
        "Available Accuracy tests for QA task are:\n",
        "\n",
        "* `min_exact_match_score`\n",
        "* `min_bleu_score`\n",
        "* `min_rouge1_score`\n",
        "* `min_rouge2_score`\n",
        "* `min_rougeL_score`\n",
        "* `min_rougeLsum_score`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bNnn_1DHZCtS"
      },
      "source": [
        "## Abstract-narrative-understanding Dataset Testing\n",
        "\n",
        "\n",
        "**Data Splits** \n",
        "- `test` - Testing set from the Abstract-narrative-understanding subset, containing 1k question and answer examples. \n",
        "- `test-tiny` - Truncated version of Abstract-narrative-understanding subset which contains 50 question answer examples.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f13UydObTDRG",
        "outputId": "d4bf1bac-1029-4c16-af8a-1c6fd9df5239"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Test Configuration : \n",
            " {\n",
            " \"model_parameters\": {\n",
            "  \"max_tokens\": 64\n",
            " },\n",
            " \"tests\": {\n",
            "  \"defaults\": {\n",
            "   \"min_pass_rate\": 1.0\n",
            "  },\n",
            "  \"robustness\": {\n",
            "   \"add_typo\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   },\n",
            "   \"lowercase\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   }\n",
            "  }\n",
            " }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "harness = Harness(\n",
        "                  task=\"question-answering\", \n",
        "                  model={\"model\": \"gpt-3.5-turbo-instruct\",\"hub\":\"openai\"}, \n",
        "                  data={\"data_source\" :\"Bigbench\",\n",
        "                        \"subset\":\"Abstract-narrative-understanding\",\n",
        "                        \"split\":\"test-tiny\"}\n",
        "                  )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8VxrRAMkBf1H"
      },
      "source": [
        "You can also set prompts and other model parameters in config. Possible parameters are:\n",
        "* `user_promt:` Promt to be given to the model.\n",
        "* `temperature:` Temperature of the model.\n",
        "* `max_tokens:` Maximum number of output tokens allowed for model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fMFVq3mCTQ7j",
        "outputId": "6b80caf2-1f7a-4aa0-a21b-3fe2d6c78468"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'tests': {'defaults': {'min_pass_rate': 0.65},\n",
              "  'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
              "   'lowercase': {'min_pass_rate': 0.6}},\n",
              "  'fairness': {'min_gender_rouge1_score': {'min_score': 0.66},\n",
              "   'min_gender_rouge2_score': {'min_score': 0.6},\n",
              "   'min_gender_rougeL_score': {'min_score': 0.66},\n",
              "   'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
              "   'max_gender_rouge1_score': {'max_score': 0.66},\n",
              "   'max_gender_rouge2_score': {'max_score': 0.6},\n",
              "   'max_gender_rougeL_score': {'max_score': 0.66},\n",
              "   'max_gender_rougeLsum_score': {'max_score': 0.66}},\n",
              "  'accuracy': {'min_exact_match_score': {'min_score': 0.8},\n",
              "   'min_rouge1_score': {'min_score': 0.8},\n",
              "   'min_rougeL_score': {'min_score': 0.8},\n",
              "   'min_bleu_score': {'min_score': 0.8},\n",
              "   'min_rouge2_score': {'min_score': 0.8},\n",
              "   'min_rougeLsum_score': {'min_score': 0.8}}}}"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.configure(\n",
        "{\n",
        " 'tests': {'defaults': {'min_pass_rate': 0.65},\n",
        "           'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
        "                          'lowercase':{'min_pass_rate': 0.60},\n",
        "                        },\n",
        "           'fairness': {\n",
        "               'min_gender_rouge1_score': {'min_score': 0.66},\n",
        "               'min_gender_rouge2_score':{'min_score': 0.60},\n",
        "               'min_gender_rougeL_score': {'min_score': 0.66},\n",
        "               'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
        "               'max_gender_rouge1_score': {'max_score': 0.66},\n",
        "               'max_gender_rouge2_score':{'max_score': 0.60},\n",
        "               'max_gender_rougeL_score': {'max_score': 0.66},\n",
        "               'max_gender_rougeLsum_score': {'max_score': 0.66},\n",
        "                        },\n",
        "          'accuracy': {'min_exact_match_score': {'min_score': 0.80},\n",
        "                        'min_rouge1_score':{'min_score': 0.80},\n",
        "                        'min_rougeL_score':{'min_score': 0.80},\n",
        "                        'min_bleu_score':{'min_score': 0.80},\n",
        "                        'min_rouge2_score':{'min_score': 0.80},\n",
        "                        'min_rougeLsum_score':{'min_score': 0.80}\n",
        "\n",
        "                        }\n",
        "          }\n",
        " }\n",
        " )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5IuCmiEBuW8"
      },
      "source": [
        "Here we have configured the harness to perform Five robustness tests and defined the minimum pass rate for each test."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nAeqBsbAB_1M"
      },
      "source": [
        "### Generating the test cases."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [],
      "source": [
        "harness.data = harness.data[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CCJxFd4nUkMN",
        "outputId": "7ae1c8c0-1c04-461b-b7a1-922a66592d10"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generating testcases...: 100%|██████████| 3/3 [00:00<00:00, 2998.07it/s]\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 580
        },
        "id": "GVriwjmeo-H_",
        "outputId": "3241474a-04e9-42db-a3e4-a9b2e7542021"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The children had been sitting outside of the k...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>THE CHILDREN HAD BEEN SITTING OUTSIDE OF THE K...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Good things come to those that wait\\n2. Onc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>He dresses in a gothic style: all black clothi...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>HE DRESSES IN A GOTHIC STYLE: ALL BLACK CLOTHI...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Never judge a book by its cover\\n2. Cut you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>She always wanted to go on a vacation to a pla...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>SHE ALWAYS WANTED TO GO ON A VACATION TO A PLA...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Live for today for tomorrow never comes\\n2....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The man who owned the little corner diner for ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>THE MAN WHO OWNED THE LITTLE CORNER DINER FOR ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Don't put new wine into old bottles\\n2. A c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Dwayne was a singer.  He went to a bar to part...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>DWAYNE WAS A SINGER. HE WENT TO A BAR TO PARTY...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. All publicity is good publicity\\n2. A proph...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The man got a new stereo. He was excited to te...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>THE MAN GOT A NEW STEREO. HE WAS EXCITED TO TE...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Do unto others as you would have them do to...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>He was desperately worried about his friends w...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>HE WAS DESPERATELY WORRIED ABOUT HIS FRIENDS W...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. A prophet is not recognized in his own land...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The homeless crowdsourced micro worker went to...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>THE HOMELESS CROWDSOURCED MICRO WORKER WENT TO...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Possession is nine points of the law\\n2. Gi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>A LATE NIGHT, TOO MUCH ALCOHOL, AND A CRAVING ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. A golden key can open any door\\n2. The cust...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>THOMAS WASN'T EXACTLY CARELESS, BUT HE WASN'T ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. To err is human; to forgive divine\\n2. Prac...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The children had been sitting outside of the k...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>the children had been sitting outside of the k...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Good things come to those that wait\\n2. Onc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>He dresses in a gothic style: all black clothi...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>he dresses in a gothic style: all black clothi...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Never judge a book by its cover\\n2. Cut you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>She always wanted to go on a vacation to a pla...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>she always wanted to go on a vacation to a pla...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Live for today for tomorrow never comes\\n2....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The man who owned the little corner diner for ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>the man who owned the little corner diner for ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Don't put new wine into old bottles\\n2. A c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Dwayne was a singer.  He went to a bar to part...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>dwayne was a singer. he went to a bar to party...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. All publicity is good publicity\\n2. A proph...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The man got a new stereo. He was excited to te...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>the man got a new stereo. he was excited to te...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Do unto others as you would have them do to...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>He was desperately worried about his friends w...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>he was desperately worried about his friends w...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. A prophet is not recognized in his own land...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The homeless crowdsourced micro worker went to...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>the homeless crowdsourced micro worker went to...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Possession is nine points of the law\\n2. Gi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. A golden key can open any door\\n2. The cust...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. To err is human; to forgive divine\\n2. Prac...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  \\\n",
              "0   robustness                   uppercase   \n",
              "1   robustness                   uppercase   \n",
              "2   robustness                   uppercase   \n",
              "3   robustness                   uppercase   \n",
              "4   robustness                   uppercase   \n",
              "5   robustness                   uppercase   \n",
              "6   robustness                   uppercase   \n",
              "7   robustness                   uppercase   \n",
              "8   robustness                   uppercase   \n",
              "9   robustness                   uppercase   \n",
              "10  robustness                   lowercase   \n",
              "11  robustness                   lowercase   \n",
              "12  robustness                   lowercase   \n",
              "13  robustness                   lowercase   \n",
              "14  robustness                   lowercase   \n",
              "15  robustness                   lowercase   \n",
              "16  robustness                   lowercase   \n",
              "17  robustness                   lowercase   \n",
              "18  robustness                   lowercase   \n",
              "19  robustness                   lowercase   \n",
              "20    fairness     min_gender_rouge1_score   \n",
              "21    fairness     min_gender_rouge1_score   \n",
              "22    fairness     min_gender_rouge1_score   \n",
              "23    fairness     min_gender_rouge2_score   \n",
              "24    fairness     min_gender_rouge2_score   \n",
              "25    fairness     min_gender_rouge2_score   \n",
              "26    fairness     min_gender_rougeL_score   \n",
              "27    fairness     min_gender_rougeL_score   \n",
              "28    fairness     min_gender_rougeL_score   \n",
              "29    fairness  min_gender_rougeLsum_score   \n",
              "30    fairness  min_gender_rougeLsum_score   \n",
              "31    fairness  min_gender_rougeLsum_score   \n",
              "32    fairness     max_gender_rouge1_score   \n",
              "33    fairness     max_gender_rouge1_score   \n",
              "34    fairness     max_gender_rouge1_score   \n",
              "35    fairness     max_gender_rouge2_score   \n",
              "36    fairness     max_gender_rouge2_score   \n",
              "37    fairness     max_gender_rouge2_score   \n",
              "38    fairness     max_gender_rougeL_score   \n",
              "39    fairness     max_gender_rougeL_score   \n",
              "40    fairness     max_gender_rougeL_score   \n",
              "41    fairness  max_gender_rougeLsum_score   \n",
              "42    fairness  max_gender_rougeLsum_score   \n",
              "43    fairness  max_gender_rougeLsum_score   \n",
              "44    accuracy       min_exact_match_score   \n",
              "45    accuracy            min_rouge1_score   \n",
              "46    accuracy            min_rougeL_score   \n",
              "47    accuracy              min_bleu_score   \n",
              "48    accuracy            min_rouge2_score   \n",
              "49    accuracy         min_rougeLsum_score   \n",
              "\n",
              "                                     original_context  \\\n",
              "0   The children had been sitting outside of the k...   \n",
              "1   He dresses in a gothic style: all black clothi...   \n",
              "2   She always wanted to go on a vacation to a pla...   \n",
              "3   The man who owned the little corner diner for ...   \n",
              "4   Dwayne was a singer.  He went to a bar to part...   \n",
              "5   The man got a new stereo. He was excited to te...   \n",
              "6   He was desperately worried about his friends w...   \n",
              "7   The homeless crowdsourced micro worker went to...   \n",
              "8   a late night, too much alcohol, and a craving ...   \n",
              "9   Thomas wasn't exactly careless, but he wasn't ...   \n",
              "10  The children had been sitting outside of the k...   \n",
              "11  He dresses in a gothic style: all black clothi...   \n",
              "12  She always wanted to go on a vacation to a pla...   \n",
              "13  The man who owned the little corner diner for ...   \n",
              "14  Dwayne was a singer.  He went to a bar to part...   \n",
              "15  The man got a new stereo. He was excited to te...   \n",
              "16  He was desperately worried about his friends w...   \n",
              "17  The homeless crowdsourced micro worker went to...   \n",
              "18  a late night, too much alcohol, and a craving ...   \n",
              "19  Thomas wasn't exactly careless, but he wasn't ...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                    original_question  \\\n",
              "0   This narrative is a good illustration of the f...   \n",
              "1   This narrative is a good illustration of the f...   \n",
              "2   This narrative is a good illustration of the f...   \n",
              "3   This narrative is a good illustration of the f...   \n",
              "4   This narrative is a good illustration of the f...   \n",
              "5   This narrative is a good illustration of the f...   \n",
              "6   This narrative is a good illustration of the f...   \n",
              "7   This narrative is a good illustration of the f...   \n",
              "8   This narrative is a good illustration of the f...   \n",
              "9   This narrative is a good illustration of the f...   \n",
              "10  This narrative is a good illustration of the f...   \n",
              "11  This narrative is a good illustration of the f...   \n",
              "12  This narrative is a good illustration of the f...   \n",
              "13  This narrative is a good illustration of the f...   \n",
              "14  This narrative is a good illustration of the f...   \n",
              "15  This narrative is a good illustration of the f...   \n",
              "16  This narrative is a good illustration of the f...   \n",
              "17  This narrative is a good illustration of the f...   \n",
              "18  This narrative is a good illustration of the f...   \n",
              "19  This narrative is a good illustration of the f...   \n",
              "20                                               male   \n",
              "21                                             female   \n",
              "22                                            unknown   \n",
              "23                                               male   \n",
              "24                                             female   \n",
              "25                                            unknown   \n",
              "26                                               male   \n",
              "27                                             female   \n",
              "28                                            unknown   \n",
              "29                                               male   \n",
              "30                                             female   \n",
              "31                                            unknown   \n",
              "32                                               male   \n",
              "33                                             female   \n",
              "34                                            unknown   \n",
              "35                                               male   \n",
              "36                                             female   \n",
              "37                                            unknown   \n",
              "38                                               male   \n",
              "39                                             female   \n",
              "40                                            unknown   \n",
              "41                                               male   \n",
              "42                                             female   \n",
              "43                                            unknown   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                    perturbed_context  \\\n",
              "0   THE CHILDREN HAD BEEN SITTING OUTSIDE OF THE K...   \n",
              "1   HE DRESSES IN A GOTHIC STYLE: ALL BLACK CLOTHI...   \n",
              "2   SHE ALWAYS WANTED TO GO ON A VACATION TO A PLA...   \n",
              "3   THE MAN WHO OWNED THE LITTLE CORNER DINER FOR ...   \n",
              "4   DWAYNE WAS A SINGER. HE WENT TO A BAR TO PARTY...   \n",
              "5   THE MAN GOT A NEW STEREO. HE WAS EXCITED TO TE...   \n",
              "6   HE WAS DESPERATELY WORRIED ABOUT HIS FRIENDS W...   \n",
              "7   THE HOMELESS CROWDSOURCED MICRO WORKER WENT TO...   \n",
              "8   A LATE NIGHT, TOO MUCH ALCOHOL, AND A CRAVING ...   \n",
              "9   THOMAS WASN'T EXACTLY CARELESS, BUT HE WASN'T ...   \n",
              "10  the children had been sitting outside of the k...   \n",
              "11  he dresses in a gothic style: all black clothi...   \n",
              "12  she always wanted to go on a vacation to a pla...   \n",
              "13  the man who owned the little corner diner for ...   \n",
              "14  dwayne was a singer. he went to a bar to party...   \n",
              "15  the man got a new stereo. he was excited to te...   \n",
              "16  he was desperately worried about his friends w...   \n",
              "17  the homeless crowdsourced micro worker went to...   \n",
              "18  a late night, too much alcohol, and a craving ...   \n",
              "19  thomas wasn't exactly careless, but he wasn't ...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                   perturbed_question  \\\n",
              "0   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "1   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "2   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "3   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "4   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "5   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "6   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "7   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "8   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "9   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "10  this narrative is a good illustration of the f...   \n",
              "11  this narrative is a good illustration of the f...   \n",
              "12  this narrative is a good illustration of the f...   \n",
              "13  this narrative is a good illustration of the f...   \n",
              "14  this narrative is a good illustration of the f...   \n",
              "15  this narrative is a good illustration of the f...   \n",
              "16  this narrative is a good illustration of the f...   \n",
              "17  this narrative is a good illustration of the f...   \n",
              "18  this narrative is a good illustration of the f...   \n",
              "19  this narrative is a good illustration of the f...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                              options  \n",
              "0   1. Good things come to those that wait\\n2. Onc...  \n",
              "1   1. Never judge a book by its cover\\n2. Cut you...  \n",
              "2   1. Live for today for tomorrow never comes\\n2....  \n",
              "3   1. Don't put new wine into old bottles\\n2. A c...  \n",
              "4   1. All publicity is good publicity\\n2. A proph...  \n",
              "5   1. Do unto others as you would have them do to...  \n",
              "6   1. A prophet is not recognized in his own land...  \n",
              "7   1. Possession is nine points of the law\\n2. Gi...  \n",
              "8   1. A golden key can open any door\\n2. The cust...  \n",
              "9   1. To err is human; to forgive divine\\n2. Prac...  \n",
              "10  1. Good things come to those that wait\\n2. Onc...  \n",
              "11  1. Never judge a book by its cover\\n2. Cut you...  \n",
              "12  1. Live for today for tomorrow never comes\\n2....  \n",
              "13  1. Don't put new wine into old bottles\\n2. A c...  \n",
              "14  1. All publicity is good publicity\\n2. A proph...  \n",
              "15  1. Do unto others as you would have them do to...  \n",
              "16  1. A prophet is not recognized in his own land...  \n",
              "17  1. Possession is nine points of the law\\n2. Gi...  \n",
              "18  1. A golden key can open any door\\n2. The cust...  \n",
              "19  1. To err is human; to forgive divine\\n2. Prac...  \n",
              "20                                                  -  \n",
              "21                                                  -  \n",
              "22                                                  -  \n",
              "23                                                  -  \n",
              "24                                                  -  \n",
              "25                                                  -  \n",
              "26                                                  -  \n",
              "27                                                  -  \n",
              "28                                                  -  \n",
              "29                                                  -  \n",
              "30                                                  -  \n",
              "31                                                  -  \n",
              "32                                                  -  \n",
              "33                                                  -  \n",
              "34                                                  -  \n",
              "35                                                  -  \n",
              "36                                                  -  \n",
              "37                                                  -  \n",
              "38                                                  -  \n",
              "39                                                  -  \n",
              "40                                                  -  \n",
              "41                                                  -  \n",
              "42                                                  -  \n",
              "43                                                  -  \n",
              "44                                                  -  \n",
              "45                                                  -  \n",
              "46                                                  -  \n",
              "47                                                  -  \n",
              "48                                                  -  \n",
              "49                                                  -  "
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.testcases()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZEWchFb8CDrk"
      },
      "source": [
        "harness.generate() method automatically generates the test cases (based on the provided configuration)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MEnLcl-OCG1O"
      },
      "source": [
        "### Running the tests"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gFEez-T0UlcC",
        "outputId": "2bb2e070-74b6-422d-ac1d-6a3fadc205d7"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Running testcases... :   0%|          | 0/50 [00:00<?, ?it/s]"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading builder script: 100%|██████████| 5.67k/5.67k [00:00<?, ?B/s]\n",
            "Downloading builder script: 100%|██████████| 5.94k/5.94k [00:00<?, ?B/s]\n",
            "Downloading extra modules: 100%|██████████| 3.34k/3.34k [00:00<00:00, 3.35MB/s]\n",
            "Running testcases... : 100%|██████████| 50/50 [03:31<00:00,  5.44s/it]"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.run()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ice4dqfCVlr"
      },
      "source": [
        "Called after harness.generate() and is to used to run all the tests.  Returns a pass/fail flag for each test."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g1NxuqveOc-t"
      },
      "source": [
        "### Generated Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 721
        },
        "id": "ZjYBONiuYJdK",
        "outputId": "e381290b-fc02-48f1-f77d-a4dec6338445"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The children had been sitting outside of the k...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE CHILDREN HAD BEEN SITTING OUTSIDE OF THE K...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Good things come to those that wait\\n2. Onc...</td>\n",
              "      <td>1. Good things come to those that wait</td>\n",
              "      <td>1. Good things come to those that wait</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>He dresses in a gothic style: all black clothi...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>HE DRESSES IN A GOTHIC STYLE: ALL BLACK CLOTHI...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Never judge a book by its cover\\n2. Cut you...</td>\n",
              "      <td>1. Never judge a book by its cover</td>\n",
              "      <td>1. Never judge a book by its cover</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>She always wanted to go on a vacation to a pla...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>SHE ALWAYS WANTED TO GO ON A VACATION TO A PLA...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Live for today for tomorrow never comes\\n2....</td>\n",
              "      <td>4. That which does not kill us makes us stronger</td>\n",
              "      <td>4. That which does not kill us makes us stronger</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The man who owned the little corner diner for ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE MAN WHO OWNED THE LITTLE CORNER DINER FOR ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Don't put new wine into old bottles\\n2. A c...</td>\n",
              "      <td>1. Don't put new wine into old bottles</td>\n",
              "      <td>4. Never judge a book by its cover</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Dwayne was a singer.  He went to a bar to part...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>DWAYNE WAS A SINGER. HE WENT TO A BAR TO PARTY...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. All publicity is good publicity\\n2. A proph...</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The man got a new stereo. He was excited to te...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE MAN GOT A NEW STEREO. HE WAS EXCITED TO TE...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Do unto others as you would have them do to...</td>\n",
              "      <td>Option 1. Do unto others as you would have th...</td>\n",
              "      <td>Option 4. People who live in glass houses sho...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>He was desperately worried about his friends w...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>HE WAS DESPERATELY WORRIED ABOUT HIS FRIENDS W...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. A prophet is not recognized in his own land...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The homeless crowdsourced micro worker went to...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE HOMELESS CROWDSOURCED MICRO WORKER WENT TO...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Possession is nine points of the law\\n2. Gi...</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>A LATE NIGHT, TOO MUCH ALCOHOL, AND A CRAVING ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. A golden key can open any door\\n2. The cust...</td>\n",
              "      <td>1. A golden key can open any door</td>\n",
              "      <td>The correct answer is option 1: A golden key ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THOMAS WASN'T EXACTLY CARELESS, BUT HE WASN'T ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. To err is human; to forgive divine\\n2. Prac...</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The children had been sitting outside of the k...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the children had been sitting outside of the k...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Good things come to those that wait\\n2. Onc...</td>\n",
              "      <td>1. Good things come to those that wait</td>\n",
              "      <td>1. Good things come to those that wait.</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>He dresses in a gothic style: all black clothi...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>he dresses in a gothic style: all black clothi...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Never judge a book by its cover\\n2. Cut you...</td>\n",
              "      <td>1. Never judge a book by its cover</td>\n",
              "      <td>Never judge a book by its cover</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>She always wanted to go on a vacation to a pla...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>she always wanted to go on a vacation to a pla...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Live for today for tomorrow never comes\\n2....</td>\n",
              "      <td>Option 4: That which does not kill us makes u...</td>\n",
              "      <td>Option 4. That which does not kill us makes u...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The man who owned the little corner diner for ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the man who owned the little corner diner for ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Don't put new wine into old bottles\\n2. A c...</td>\n",
              "      <td>1. Don't put new wine into old bottles</td>\n",
              "      <td>Option 1. Don't put new wine into old bottles</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Dwayne was a singer.  He went to a bar to part...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>dwayne was a singer. he went to a bar to party...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. All publicity is good publicity\\n2. A proph...</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The man got a new stereo. He was excited to te...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the man got a new stereo. he was excited to te...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Do unto others as you would have them do to...</td>\n",
              "      <td>Option 1. Do unto others as you would have th...</td>\n",
              "      <td>1. Do unto others as you would have them do t...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>He was desperately worried about his friends w...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>he was desperately worried about his friends w...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. A prophet is not recognized in his own land...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The homeless crowdsourced micro worker went to...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the homeless crowdsourced micro worker went to...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Possession is nine points of the law\\n2. Gi...</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. A golden key can open any door\\n2. The cust...</td>\n",
              "      <td>Option 2. The customer is always right.</td>\n",
              "      <td>1. A golden key can open any door</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. To err is human; to forgive divine\\n2. Prac...</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.81385</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.923077</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.838035</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.81385</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.923077</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.838035</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.1</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.761435</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.762977</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.556467</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.742362</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.7589</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  \\\n",
              "0   robustness                   uppercase   \n",
              "1   robustness                   uppercase   \n",
              "2   robustness                   uppercase   \n",
              "3   robustness                   uppercase   \n",
              "4   robustness                   uppercase   \n",
              "5   robustness                   uppercase   \n",
              "6   robustness                   uppercase   \n",
              "7   robustness                   uppercase   \n",
              "8   robustness                   uppercase   \n",
              "9   robustness                   uppercase   \n",
              "10  robustness                   lowercase   \n",
              "11  robustness                   lowercase   \n",
              "12  robustness                   lowercase   \n",
              "13  robustness                   lowercase   \n",
              "14  robustness                   lowercase   \n",
              "15  robustness                   lowercase   \n",
              "16  robustness                   lowercase   \n",
              "17  robustness                   lowercase   \n",
              "18  robustness                   lowercase   \n",
              "19  robustness                   lowercase   \n",
              "20    fairness     min_gender_rouge1_score   \n",
              "21    fairness     min_gender_rouge1_score   \n",
              "22    fairness     min_gender_rouge1_score   \n",
              "23    fairness     min_gender_rouge2_score   \n",
              "24    fairness     min_gender_rouge2_score   \n",
              "25    fairness     min_gender_rouge2_score   \n",
              "26    fairness     min_gender_rougeL_score   \n",
              "27    fairness     min_gender_rougeL_score   \n",
              "28    fairness     min_gender_rougeL_score   \n",
              "29    fairness  min_gender_rougeLsum_score   \n",
              "30    fairness  min_gender_rougeLsum_score   \n",
              "31    fairness  min_gender_rougeLsum_score   \n",
              "32    fairness     max_gender_rouge1_score   \n",
              "33    fairness     max_gender_rouge1_score   \n",
              "34    fairness     max_gender_rouge1_score   \n",
              "35    fairness     max_gender_rouge2_score   \n",
              "36    fairness     max_gender_rouge2_score   \n",
              "37    fairness     max_gender_rouge2_score   \n",
              "38    fairness     max_gender_rougeL_score   \n",
              "39    fairness     max_gender_rougeL_score   \n",
              "40    fairness     max_gender_rougeL_score   \n",
              "41    fairness  max_gender_rougeLsum_score   \n",
              "42    fairness  max_gender_rougeLsum_score   \n",
              "43    fairness  max_gender_rougeLsum_score   \n",
              "44    accuracy       min_exact_match_score   \n",
              "45    accuracy            min_rouge1_score   \n",
              "46    accuracy            min_rougeL_score   \n",
              "47    accuracy              min_bleu_score   \n",
              "48    accuracy            min_rouge2_score   \n",
              "49    accuracy         min_rougeLsum_score   \n",
              "\n",
              "                                     original_context  \\\n",
              "0   The children had been sitting outside of the k...   \n",
              "1   He dresses in a gothic style: all black clothi...   \n",
              "2   She always wanted to go on a vacation to a pla...   \n",
              "3   The man who owned the little corner diner for ...   \n",
              "4   Dwayne was a singer.  He went to a bar to part...   \n",
              "5   The man got a new stereo. He was excited to te...   \n",
              "6   He was desperately worried about his friends w...   \n",
              "7   The homeless crowdsourced micro worker went to...   \n",
              "8   a late night, too much alcohol, and a craving ...   \n",
              "9   Thomas wasn't exactly careless, but he wasn't ...   \n",
              "10  The children had been sitting outside of the k...   \n",
              "11  He dresses in a gothic style: all black clothi...   \n",
              "12  She always wanted to go on a vacation to a pla...   \n",
              "13  The man who owned the little corner diner for ...   \n",
              "14  Dwayne was a singer.  He went to a bar to part...   \n",
              "15  The man got a new stereo. He was excited to te...   \n",
              "16  He was desperately worried about his friends w...   \n",
              "17  The homeless crowdsourced micro worker went to...   \n",
              "18  a late night, too much alcohol, and a craving ...   \n",
              "19  Thomas wasn't exactly careless, but he wasn't ...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                    original_question test_case  \\\n",
              "0   This narrative is a good illustration of the f...         -   \n",
              "1   This narrative is a good illustration of the f...         -   \n",
              "2   This narrative is a good illustration of the f...         -   \n",
              "3   This narrative is a good illustration of the f...         -   \n",
              "4   This narrative is a good illustration of the f...         -   \n",
              "5   This narrative is a good illustration of the f...         -   \n",
              "6   This narrative is a good illustration of the f...         -   \n",
              "7   This narrative is a good illustration of the f...         -   \n",
              "8   This narrative is a good illustration of the f...         -   \n",
              "9   This narrative is a good illustration of the f...         -   \n",
              "10  This narrative is a good illustration of the f...         -   \n",
              "11  This narrative is a good illustration of the f...         -   \n",
              "12  This narrative is a good illustration of the f...         -   \n",
              "13  This narrative is a good illustration of the f...         -   \n",
              "14  This narrative is a good illustration of the f...         -   \n",
              "15  This narrative is a good illustration of the f...         -   \n",
              "16  This narrative is a good illustration of the f...         -   \n",
              "17  This narrative is a good illustration of the f...         -   \n",
              "18  This narrative is a good illustration of the f...         -   \n",
              "19  This narrative is a good illustration of the f...         -   \n",
              "20                                                  -      male   \n",
              "21                                                  -    female   \n",
              "22                                                  -   unknown   \n",
              "23                                                  -      male   \n",
              "24                                                  -    female   \n",
              "25                                                  -   unknown   \n",
              "26                                                  -      male   \n",
              "27                                                  -    female   \n",
              "28                                                  -   unknown   \n",
              "29                                                  -      male   \n",
              "30                                                  -    female   \n",
              "31                                                  -   unknown   \n",
              "32                                                  -      male   \n",
              "33                                                  -    female   \n",
              "34                                                  -   unknown   \n",
              "35                                                  -      male   \n",
              "36                                                  -    female   \n",
              "37                                                  -   unknown   \n",
              "38                                                  -      male   \n",
              "39                                                  -    female   \n",
              "40                                                  -   unknown   \n",
              "41                                                  -      male   \n",
              "42                                                  -    female   \n",
              "43                                                  -   unknown   \n",
              "44                                                  -         -   \n",
              "45                                                  -         -   \n",
              "46                                                  -         -   \n",
              "47                                                  -         -   \n",
              "48                                                  -         -   \n",
              "49                                                  -         -   \n",
              "\n",
              "                                    perturbed_context  \\\n",
              "0   THE CHILDREN HAD BEEN SITTING OUTSIDE OF THE K...   \n",
              "1   HE DRESSES IN A GOTHIC STYLE: ALL BLACK CLOTHI...   \n",
              "2   SHE ALWAYS WANTED TO GO ON A VACATION TO A PLA...   \n",
              "3   THE MAN WHO OWNED THE LITTLE CORNER DINER FOR ...   \n",
              "4   DWAYNE WAS A SINGER. HE WENT TO A BAR TO PARTY...   \n",
              "5   THE MAN GOT A NEW STEREO. HE WAS EXCITED TO TE...   \n",
              "6   HE WAS DESPERATELY WORRIED ABOUT HIS FRIENDS W...   \n",
              "7   THE HOMELESS CROWDSOURCED MICRO WORKER WENT TO...   \n",
              "8   A LATE NIGHT, TOO MUCH ALCOHOL, AND A CRAVING ...   \n",
              "9   THOMAS WASN'T EXACTLY CARELESS, BUT HE WASN'T ...   \n",
              "10  the children had been sitting outside of the k...   \n",
              "11  he dresses in a gothic style: all black clothi...   \n",
              "12  she always wanted to go on a vacation to a pla...   \n",
              "13  the man who owned the little corner diner for ...   \n",
              "14  dwayne was a singer. he went to a bar to party...   \n",
              "15  the man got a new stereo. he was excited to te...   \n",
              "16  he was desperately worried about his friends w...   \n",
              "17  the homeless crowdsourced micro worker went to...   \n",
              "18  a late night, too much alcohol, and a craving ...   \n",
              "19  thomas wasn't exactly careless, but he wasn't ...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                   perturbed_question  \\\n",
              "0   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "1   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "2   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "3   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "4   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "5   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "6   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "7   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "8   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "9   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "10  this narrative is a good illustration of the f...   \n",
              "11  this narrative is a good illustration of the f...   \n",
              "12  this narrative is a good illustration of the f...   \n",
              "13  this narrative is a good illustration of the f...   \n",
              "14  this narrative is a good illustration of the f...   \n",
              "15  this narrative is a good illustration of the f...   \n",
              "16  this narrative is a good illustration of the f...   \n",
              "17  this narrative is a good illustration of the f...   \n",
              "18  this narrative is a good illustration of the f...   \n",
              "19  this narrative is a good illustration of the f...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                              options  \\\n",
              "0   1. Good things come to those that wait\\n2. Onc...   \n",
              "1   1. Never judge a book by its cover\\n2. Cut you...   \n",
              "2   1. Live for today for tomorrow never comes\\n2....   \n",
              "3   1. Don't put new wine into old bottles\\n2. A c...   \n",
              "4   1. All publicity is good publicity\\n2. A proph...   \n",
              "5   1. Do unto others as you would have them do to...   \n",
              "6   1. A prophet is not recognized in his own land...   \n",
              "7   1. Possession is nine points of the law\\n2. Gi...   \n",
              "8   1. A golden key can open any door\\n2. The cust...   \n",
              "9   1. To err is human; to forgive divine\\n2. Prac...   \n",
              "10  1. Good things come to those that wait\\n2. Onc...   \n",
              "11  1. Never judge a book by its cover\\n2. Cut you...   \n",
              "12  1. Live for today for tomorrow never comes\\n2....   \n",
              "13  1. Don't put new wine into old bottles\\n2. A c...   \n",
              "14  1. All publicity is good publicity\\n2. A proph...   \n",
              "15  1. Do unto others as you would have them do to...   \n",
              "16  1. A prophet is not recognized in his own land...   \n",
              "17  1. Possession is nine points of the law\\n2. Gi...   \n",
              "18  1. A golden key can open any door\\n2. The cust...   \n",
              "19  1. To err is human; to forgive divine\\n2. Prac...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                      expected_result  \\\n",
              "0              1. Good things come to those that wait   \n",
              "1                  1. Never judge a book by its cover   \n",
              "2    4. That which does not kill us makes us stronger   \n",
              "3              1. Don't put new wine into old bottles   \n",
              "4                  1. All publicity is good publicity   \n",
              "5    Option 1. Do unto others as you would have th...   \n",
              "6    2. Success has many fathers, while failure is...   \n",
              "7             1. Possession is nine points of the law   \n",
              "8                   1. A golden key can open any door   \n",
              "9               1. To err is human; to forgive divine   \n",
              "10             1. Good things come to those that wait   \n",
              "11                 1. Never judge a book by its cover   \n",
              "12   Option 4: That which does not kill us makes u...   \n",
              "13             1. Don't put new wine into old bottles   \n",
              "14                 1. All publicity is good publicity   \n",
              "15   Option 1. Do unto others as you would have th...   \n",
              "16   2. Success has many fathers, while failure is...   \n",
              "17            1. Possession is nine points of the law   \n",
              "18            Option 2. The customer is always right.   \n",
              "19              1. To err is human; to forgive divine   \n",
              "20                                               0.66   \n",
              "21                                               0.66   \n",
              "22                                               0.66   \n",
              "23                                                0.6   \n",
              "24                                                0.6   \n",
              "25                                                0.6   \n",
              "26                                               0.66   \n",
              "27                                               0.66   \n",
              "28                                               0.66   \n",
              "29                                               0.66   \n",
              "30                                               0.66   \n",
              "31                                               0.66   \n",
              "32                                               0.66   \n",
              "33                                               0.66   \n",
              "34                                               0.66   \n",
              "35                                                0.6   \n",
              "36                                                0.6   \n",
              "37                                                0.6   \n",
              "38                                               0.66   \n",
              "39                                               0.66   \n",
              "40                                               0.66   \n",
              "41                                               0.66   \n",
              "42                                               0.66   \n",
              "43                                               0.66   \n",
              "44                                                0.8   \n",
              "45                                                0.8   \n",
              "46                                                0.8   \n",
              "47                                                0.8   \n",
              "48                                                0.8   \n",
              "49                                                0.8   \n",
              "\n",
              "                                        actual_result   pass  \n",
              "0              1. Good things come to those that wait   True  \n",
              "1                  1. Never judge a book by its cover   True  \n",
              "2    4. That which does not kill us makes us stronger   True  \n",
              "3                  4. Never judge a book by its cover  False  \n",
              "4                  1. All publicity is good publicity   True  \n",
              "5    Option 4. People who live in glass houses sho...  False  \n",
              "6    2. Success has many fathers, while failure is...   True  \n",
              "7             1. Possession is nine points of the law   True  \n",
              "8    The correct answer is option 1: A golden key ...   True  \n",
              "9               1. To err is human; to forgive divine   True  \n",
              "10            1. Good things come to those that wait.   True  \n",
              "11                    Never judge a book by its cover   True  \n",
              "12   Option 4. That which does not kill us makes u...   True  \n",
              "13      Option 1. Don't put new wine into old bottles  False  \n",
              "14                 1. All publicity is good publicity   True  \n",
              "15   1. Do unto others as you would have them do t...   True  \n",
              "16   2. Success has many fathers, while failure is...   True  \n",
              "17            1. Possession is nine points of the law   True  \n",
              "18                  1. A golden key can open any door  False  \n",
              "19              1. To err is human; to forgive divine   True  \n",
              "20                                           0.836108   True  \n",
              "21                                                0.0  False  \n",
              "22                                           0.933333   True  \n",
              "23                                            0.81385   True  \n",
              "24                                                0.0  False  \n",
              "25                                           0.923077   True  \n",
              "26                                           0.838035   True  \n",
              "27                                                0.0  False  \n",
              "28                                           0.933333   True  \n",
              "29                                           0.836108   True  \n",
              "30                                                0.0  False  \n",
              "31                                           0.933333   True  \n",
              "32                                           0.836108  False  \n",
              "33                                                0.0   True  \n",
              "34                                           0.933333  False  \n",
              "35                                            0.81385  False  \n",
              "36                                                0.0   True  \n",
              "37                                           0.923077  False  \n",
              "38                                           0.838035  False  \n",
              "39                                                0.0   True  \n",
              "40                                           0.933333  False  \n",
              "41                                           0.836108  False  \n",
              "42                                                0.0   True  \n",
              "43                                           0.933333  False  \n",
              "44                                                0.1  False  \n",
              "45                                           0.761435  False  \n",
              "46                                           0.762977  False  \n",
              "47                                           0.556467  False  \n",
              "48                                           0.742362  False  \n",
              "49                                             0.7589  False  "
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generated_results()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gl5QGV9pCZfz"
      },
      "source": [
        "This method returns the generated results in the form of a pandas dataframe, which provides a convenient and easy-to-use format for working with the test results. You can use this method to quickly identify the test cases that failed and to determine where fixes are needed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "MQkbrUmkd1bF"
      },
      "outputs": [],
      "source": [
        "df=harness.generated_results()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pJ3DUIUWeJmP"
      },
      "source": [
        "### Generated Results for Robustness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 929
        },
        "id": "pI52zDVtd3ER",
        "outputId": "b67061b7-0c0c-448f-95b0-ada2d9f98a11"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The children had been sitting outside of the k...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE CHILDREN HAD BEEN SITTING OUTSIDE OF THE K...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Good things come to those that wait\\n2. Onc...</td>\n",
              "      <td>1. Good things come to those that wait</td>\n",
              "      <td>1. Good things come to those that wait</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>He dresses in a gothic style: all black clothi...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>HE DRESSES IN A GOTHIC STYLE: ALL BLACK CLOTHI...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Never judge a book by its cover\\n2. Cut you...</td>\n",
              "      <td>1. Never judge a book by its cover</td>\n",
              "      <td>1. Never judge a book by its cover</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>She always wanted to go on a vacation to a pla...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>SHE ALWAYS WANTED TO GO ON A VACATION TO A PLA...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Live for today for tomorrow never comes\\n2....</td>\n",
              "      <td>4. That which does not kill us makes us stronger</td>\n",
              "      <td>4. That which does not kill us makes us stronger</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The man who owned the little corner diner for ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE MAN WHO OWNED THE LITTLE CORNER DINER FOR ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Don't put new wine into old bottles\\n2. A c...</td>\n",
              "      <td>1. Don't put new wine into old bottles</td>\n",
              "      <td>4. Never judge a book by its cover</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Dwayne was a singer.  He went to a bar to part...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>DWAYNE WAS A SINGER. HE WENT TO A BAR TO PARTY...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. All publicity is good publicity\\n2. A proph...</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The man got a new stereo. He was excited to te...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE MAN GOT A NEW STEREO. HE WAS EXCITED TO TE...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Do unto others as you would have them do to...</td>\n",
              "      <td>Option 1. Do unto others as you would have th...</td>\n",
              "      <td>Option 4. People who live in glass houses sho...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>He was desperately worried about his friends w...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>HE WAS DESPERATELY WORRIED ABOUT HIS FRIENDS W...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. A prophet is not recognized in his own land...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The homeless crowdsourced micro worker went to...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE HOMELESS CROWDSOURCED MICRO WORKER WENT TO...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. Possession is nine points of the law\\n2. Gi...</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>A LATE NIGHT, TOO MUCH ALCOHOL, AND A CRAVING ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. A golden key can open any door\\n2. The cust...</td>\n",
              "      <td>1. A golden key can open any door</td>\n",
              "      <td>The correct answer is option 1: A golden key ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>THOMAS WASN'T EXACTLY CARELESS, BUT HE WASN'T ...</td>\n",
              "      <td>THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...</td>\n",
              "      <td>1. To err is human; to forgive divine\\n2. Prac...</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The children had been sitting outside of the k...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the children had been sitting outside of the k...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Good things come to those that wait\\n2. Onc...</td>\n",
              "      <td>1. Good things come to those that wait</td>\n",
              "      <td>1. Good things come to those that wait.</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>He dresses in a gothic style: all black clothi...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>he dresses in a gothic style: all black clothi...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Never judge a book by its cover\\n2. Cut you...</td>\n",
              "      <td>1. Never judge a book by its cover</td>\n",
              "      <td>Never judge a book by its cover</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>She always wanted to go on a vacation to a pla...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>she always wanted to go on a vacation to a pla...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Live for today for tomorrow never comes\\n2....</td>\n",
              "      <td>Option 4: That which does not kill us makes u...</td>\n",
              "      <td>Option 4. That which does not kill us makes u...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The man who owned the little corner diner for ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the man who owned the little corner diner for ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Don't put new wine into old bottles\\n2. A c...</td>\n",
              "      <td>1. Don't put new wine into old bottles</td>\n",
              "      <td>Option 1. Don't put new wine into old bottles</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Dwayne was a singer.  He went to a bar to part...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>dwayne was a singer. he went to a bar to party...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. All publicity is good publicity\\n2. A proph...</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>1. All publicity is good publicity</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The man got a new stereo. He was excited to te...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the man got a new stereo. he was excited to te...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Do unto others as you would have them do to...</td>\n",
              "      <td>Option 1. Do unto others as you would have th...</td>\n",
              "      <td>1. Do unto others as you would have them do t...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>He was desperately worried about his friends w...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>he was desperately worried about his friends w...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. A prophet is not recognized in his own land...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>2. Success has many fathers, while failure is...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The homeless crowdsourced micro worker went to...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>the homeless crowdsourced micro worker went to...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. Possession is nine points of the law\\n2. Gi...</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>1. Possession is nine points of the law</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>a late night, too much alcohol, and a craving ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. A golden key can open any door\\n2. The cust...</td>\n",
              "      <td>Option 2. The customer is always right.</td>\n",
              "      <td>1. A golden key can open any door</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>This narrative is a good illustration of the f...</td>\n",
              "      <td>-</td>\n",
              "      <td>thomas wasn't exactly careless, but he wasn't ...</td>\n",
              "      <td>this narrative is a good illustration of the f...</td>\n",
              "      <td>1. To err is human; to forgive divine\\n2. Prac...</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>1. To err is human; to forgive divine</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category  test_type                                   original_context  \\\n",
              "0   robustness  uppercase  The children had been sitting outside of the k...   \n",
              "1   robustness  uppercase  He dresses in a gothic style: all black clothi...   \n",
              "2   robustness  uppercase  She always wanted to go on a vacation to a pla...   \n",
              "3   robustness  uppercase  The man who owned the little corner diner for ...   \n",
              "4   robustness  uppercase  Dwayne was a singer.  He went to a bar to part...   \n",
              "5   robustness  uppercase  The man got a new stereo. He was excited to te...   \n",
              "6   robustness  uppercase  He was desperately worried about his friends w...   \n",
              "7   robustness  uppercase  The homeless crowdsourced micro worker went to...   \n",
              "8   robustness  uppercase  a late night, too much alcohol, and a craving ...   \n",
              "9   robustness  uppercase  Thomas wasn't exactly careless, but he wasn't ...   \n",
              "10  robustness  lowercase  The children had been sitting outside of the k...   \n",
              "11  robustness  lowercase  He dresses in a gothic style: all black clothi...   \n",
              "12  robustness  lowercase  She always wanted to go on a vacation to a pla...   \n",
              "13  robustness  lowercase  The man who owned the little corner diner for ...   \n",
              "14  robustness  lowercase  Dwayne was a singer.  He went to a bar to part...   \n",
              "15  robustness  lowercase  The man got a new stereo. He was excited to te...   \n",
              "16  robustness  lowercase  He was desperately worried about his friends w...   \n",
              "17  robustness  lowercase  The homeless crowdsourced micro worker went to...   \n",
              "18  robustness  lowercase  a late night, too much alcohol, and a craving ...   \n",
              "19  robustness  lowercase  Thomas wasn't exactly careless, but he wasn't ...   \n",
              "\n",
              "                                    original_question test_case  \\\n",
              "0   This narrative is a good illustration of the f...         -   \n",
              "1   This narrative is a good illustration of the f...         -   \n",
              "2   This narrative is a good illustration of the f...         -   \n",
              "3   This narrative is a good illustration of the f...         -   \n",
              "4   This narrative is a good illustration of the f...         -   \n",
              "5   This narrative is a good illustration of the f...         -   \n",
              "6   This narrative is a good illustration of the f...         -   \n",
              "7   This narrative is a good illustration of the f...         -   \n",
              "8   This narrative is a good illustration of the f...         -   \n",
              "9   This narrative is a good illustration of the f...         -   \n",
              "10  This narrative is a good illustration of the f...         -   \n",
              "11  This narrative is a good illustration of the f...         -   \n",
              "12  This narrative is a good illustration of the f...         -   \n",
              "13  This narrative is a good illustration of the f...         -   \n",
              "14  This narrative is a good illustration of the f...         -   \n",
              "15  This narrative is a good illustration of the f...         -   \n",
              "16  This narrative is a good illustration of the f...         -   \n",
              "17  This narrative is a good illustration of the f...         -   \n",
              "18  This narrative is a good illustration of the f...         -   \n",
              "19  This narrative is a good illustration of the f...         -   \n",
              "\n",
              "                                    perturbed_context  \\\n",
              "0   THE CHILDREN HAD BEEN SITTING OUTSIDE OF THE K...   \n",
              "1   HE DRESSES IN A GOTHIC STYLE: ALL BLACK CLOTHI...   \n",
              "2   SHE ALWAYS WANTED TO GO ON A VACATION TO A PLA...   \n",
              "3   THE MAN WHO OWNED THE LITTLE CORNER DINER FOR ...   \n",
              "4   DWAYNE WAS A SINGER. HE WENT TO A BAR TO PARTY...   \n",
              "5   THE MAN GOT A NEW STEREO. HE WAS EXCITED TO TE...   \n",
              "6   HE WAS DESPERATELY WORRIED ABOUT HIS FRIENDS W...   \n",
              "7   THE HOMELESS CROWDSOURCED MICRO WORKER WENT TO...   \n",
              "8   A LATE NIGHT, TOO MUCH ALCOHOL, AND A CRAVING ...   \n",
              "9   THOMAS WASN'T EXACTLY CARELESS, BUT HE WASN'T ...   \n",
              "10  the children had been sitting outside of the k...   \n",
              "11  he dresses in a gothic style: all black clothi...   \n",
              "12  she always wanted to go on a vacation to a pla...   \n",
              "13  the man who owned the little corner diner for ...   \n",
              "14  dwayne was a singer. he went to a bar to party...   \n",
              "15  the man got a new stereo. he was excited to te...   \n",
              "16  he was desperately worried about his friends w...   \n",
              "17  the homeless crowdsourced micro worker went to...   \n",
              "18  a late night, too much alcohol, and a craving ...   \n",
              "19  thomas wasn't exactly careless, but he wasn't ...   \n",
              "\n",
              "                                   perturbed_question  \\\n",
              "0   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "1   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "2   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "3   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "4   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "5   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "6   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "7   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "8   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "9   THIS NARRATIVE IS A GOOD ILLUSTRATION OF THE F...   \n",
              "10  this narrative is a good illustration of the f...   \n",
              "11  this narrative is a good illustration of the f...   \n",
              "12  this narrative is a good illustration of the f...   \n",
              "13  this narrative is a good illustration of the f...   \n",
              "14  this narrative is a good illustration of the f...   \n",
              "15  this narrative is a good illustration of the f...   \n",
              "16  this narrative is a good illustration of the f...   \n",
              "17  this narrative is a good illustration of the f...   \n",
              "18  this narrative is a good illustration of the f...   \n",
              "19  this narrative is a good illustration of the f...   \n",
              "\n",
              "                                              options  \\\n",
              "0   1. Good things come to those that wait\\n2. Onc...   \n",
              "1   1. Never judge a book by its cover\\n2. Cut you...   \n",
              "2   1. Live for today for tomorrow never comes\\n2....   \n",
              "3   1. Don't put new wine into old bottles\\n2. A c...   \n",
              "4   1. All publicity is good publicity\\n2. A proph...   \n",
              "5   1. Do unto others as you would have them do to...   \n",
              "6   1. A prophet is not recognized in his own land...   \n",
              "7   1. Possession is nine points of the law\\n2. Gi...   \n",
              "8   1. A golden key can open any door\\n2. The cust...   \n",
              "9   1. To err is human; to forgive divine\\n2. Prac...   \n",
              "10  1. Good things come to those that wait\\n2. Onc...   \n",
              "11  1. Never judge a book by its cover\\n2. Cut you...   \n",
              "12  1. Live for today for tomorrow never comes\\n2....   \n",
              "13  1. Don't put new wine into old bottles\\n2. A c...   \n",
              "14  1. All publicity is good publicity\\n2. A proph...   \n",
              "15  1. Do unto others as you would have them do to...   \n",
              "16  1. A prophet is not recognized in his own land...   \n",
              "17  1. Possession is nine points of the law\\n2. Gi...   \n",
              "18  1. A golden key can open any door\\n2. The cust...   \n",
              "19  1. To err is human; to forgive divine\\n2. Prac...   \n",
              "\n",
              "                                      expected_result  \\\n",
              "0              1. Good things come to those that wait   \n",
              "1                  1. Never judge a book by its cover   \n",
              "2    4. That which does not kill us makes us stronger   \n",
              "3              1. Don't put new wine into old bottles   \n",
              "4                  1. All publicity is good publicity   \n",
              "5    Option 1. Do unto others as you would have th...   \n",
              "6    2. Success has many fathers, while failure is...   \n",
              "7             1. Possession is nine points of the law   \n",
              "8                   1. A golden key can open any door   \n",
              "9               1. To err is human; to forgive divine   \n",
              "10             1. Good things come to those that wait   \n",
              "11                 1. Never judge a book by its cover   \n",
              "12   Option 4: That which does not kill us makes u...   \n",
              "13             1. Don't put new wine into old bottles   \n",
              "14                 1. All publicity is good publicity   \n",
              "15   Option 1. Do unto others as you would have th...   \n",
              "16   2. Success has many fathers, while failure is...   \n",
              "17            1. Possession is nine points of the law   \n",
              "18            Option 2. The customer is always right.   \n",
              "19              1. To err is human; to forgive divine   \n",
              "\n",
              "                                        actual_result   pass  \n",
              "0              1. Good things come to those that wait   True  \n",
              "1                  1. Never judge a book by its cover   True  \n",
              "2    4. That which does not kill us makes us stronger   True  \n",
              "3                  4. Never judge a book by its cover  False  \n",
              "4                  1. All publicity is good publicity   True  \n",
              "5    Option 4. People who live in glass houses sho...  False  \n",
              "6    2. Success has many fathers, while failure is...   True  \n",
              "7             1. Possession is nine points of the law   True  \n",
              "8    The correct answer is option 1: A golden key ...   True  \n",
              "9               1. To err is human; to forgive divine   True  \n",
              "10            1. Good things come to those that wait.   True  \n",
              "11                    Never judge a book by its cover   True  \n",
              "12   Option 4. That which does not kill us makes u...   True  \n",
              "13      Option 1. Don't put new wine into old bottles  False  \n",
              "14                 1. All publicity is good publicity   True  \n",
              "15   1. Do unto others as you would have them do t...   True  \n",
              "16   2. Success has many fathers, while failure is...   True  \n",
              "17            1. Possession is nine points of the law   True  \n",
              "18                  1. A golden key can open any door  False  \n",
              "19              1. To err is human; to forgive divine   True  "
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"robustness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-dIEPhi-eYza"
      },
      "source": [
        "### Generated Results for fairness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 822
        },
        "id": "NYRvvm5QeWyC",
        "outputId": "8cd338a4-689d-40e0-da21-2bec6967fdf6"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.81385</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.923077</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.838035</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.81385</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.923077</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.838035</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.836108</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.933333</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    category                   test_type original_context original_question  \\\n",
              "20  fairness     min_gender_rouge1_score                -                 -   \n",
              "21  fairness     min_gender_rouge1_score                -                 -   \n",
              "22  fairness     min_gender_rouge1_score                -                 -   \n",
              "23  fairness     min_gender_rouge2_score                -                 -   \n",
              "24  fairness     min_gender_rouge2_score                -                 -   \n",
              "25  fairness     min_gender_rouge2_score                -                 -   \n",
              "26  fairness     min_gender_rougeL_score                -                 -   \n",
              "27  fairness     min_gender_rougeL_score                -                 -   \n",
              "28  fairness     min_gender_rougeL_score                -                 -   \n",
              "29  fairness  min_gender_rougeLsum_score                -                 -   \n",
              "30  fairness  min_gender_rougeLsum_score                -                 -   \n",
              "31  fairness  min_gender_rougeLsum_score                -                 -   \n",
              "32  fairness     max_gender_rouge1_score                -                 -   \n",
              "33  fairness     max_gender_rouge1_score                -                 -   \n",
              "34  fairness     max_gender_rouge1_score                -                 -   \n",
              "35  fairness     max_gender_rouge2_score                -                 -   \n",
              "36  fairness     max_gender_rouge2_score                -                 -   \n",
              "37  fairness     max_gender_rouge2_score                -                 -   \n",
              "38  fairness     max_gender_rougeL_score                -                 -   \n",
              "39  fairness     max_gender_rougeL_score                -                 -   \n",
              "40  fairness     max_gender_rougeL_score                -                 -   \n",
              "41  fairness  max_gender_rougeLsum_score                -                 -   \n",
              "42  fairness  max_gender_rougeLsum_score                -                 -   \n",
              "43  fairness  max_gender_rougeLsum_score                -                 -   \n",
              "\n",
              "   test_case perturbed_context perturbed_question options expected_result  \\\n",
              "20      male                 -                  -       -            0.66   \n",
              "21    female                 -                  -       -            0.66   \n",
              "22   unknown                 -                  -       -            0.66   \n",
              "23      male                 -                  -       -             0.6   \n",
              "24    female                 -                  -       -             0.6   \n",
              "25   unknown                 -                  -       -             0.6   \n",
              "26      male                 -                  -       -            0.66   \n",
              "27    female                 -                  -       -            0.66   \n",
              "28   unknown                 -                  -       -            0.66   \n",
              "29      male                 -                  -       -            0.66   \n",
              "30    female                 -                  -       -            0.66   \n",
              "31   unknown                 -                  -       -            0.66   \n",
              "32      male                 -                  -       -            0.66   \n",
              "33    female                 -                  -       -            0.66   \n",
              "34   unknown                 -                  -       -            0.66   \n",
              "35      male                 -                  -       -             0.6   \n",
              "36    female                 -                  -       -             0.6   \n",
              "37   unknown                 -                  -       -             0.6   \n",
              "38      male                 -                  -       -            0.66   \n",
              "39    female                 -                  -       -            0.66   \n",
              "40   unknown                 -                  -       -            0.66   \n",
              "41      male                 -                  -       -            0.66   \n",
              "42    female                 -                  -       -            0.66   \n",
              "43   unknown                 -                  -       -            0.66   \n",
              "\n",
              "   actual_result   pass  \n",
              "20      0.836108   True  \n",
              "21           0.0  False  \n",
              "22      0.933333   True  \n",
              "23       0.81385   True  \n",
              "24           0.0  False  \n",
              "25      0.923077   True  \n",
              "26      0.838035   True  \n",
              "27           0.0  False  \n",
              "28      0.933333   True  \n",
              "29      0.836108   True  \n",
              "30           0.0  False  \n",
              "31      0.933333   True  \n",
              "32      0.836108  False  \n",
              "33           0.0   True  \n",
              "34      0.933333  False  \n",
              "35       0.81385  False  \n",
              "36           0.0   True  \n",
              "37      0.923077  False  \n",
              "38      0.838035  False  \n",
              "39           0.0   True  \n",
              "40      0.933333  False  \n",
              "41      0.836108  False  \n",
              "42           0.0   True  \n",
              "43      0.933333  False  "
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"fairness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WZef_QAcenUr"
      },
      "source": [
        "### Generated Results for accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "R2VEVY0leqL9",
        "outputId": "03bc1227-76e7-4578-93a8-4f363841d400"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.1</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.761435</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.762977</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.556467</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.742362</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.7589</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    category              test_type original_context original_question  \\\n",
              "44  accuracy  min_exact_match_score                -                 -   \n",
              "45  accuracy       min_rouge1_score                -                 -   \n",
              "46  accuracy       min_rougeL_score                -                 -   \n",
              "47  accuracy         min_bleu_score                -                 -   \n",
              "48  accuracy       min_rouge2_score                -                 -   \n",
              "49  accuracy    min_rougeLsum_score                -                 -   \n",
              "\n",
              "   test_case perturbed_context perturbed_question options expected_result  \\\n",
              "44         -                 -                  -       -             0.8   \n",
              "45         -                 -                  -       -             0.8   \n",
              "46         -                 -                  -       -             0.8   \n",
              "47         -                 -                  -       -             0.8   \n",
              "48         -                 -                  -       -             0.8   \n",
              "49         -                 -                  -       -             0.8   \n",
              "\n",
              "   actual_result   pass  \n",
              "44           0.1  False  \n",
              "45      0.761435  False  \n",
              "46      0.762977  False  \n",
              "47      0.556467  False  \n",
              "48      0.742362  False  \n",
              "49        0.7589  False  "
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"accuracy\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FOSn8Fi6eRW1"
      },
      "source": [
        "### Generated Results For bias"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9fBgU33hCb2K"
      },
      "source": [
        "### Final Results\n",
        "\n",
        "We can call `.report()` which summarizes the results giving information about pass and fail counts and overall test pass/fail flag."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 551
        },
        "id": "nDmRw1AeUqIl",
        "outputId": "568bf1e9-08fc-45b6-9237-f09724d3d397"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>fail_count</th>\n",
              "      <th>pass_count</th>\n",
              "      <th>pass_rate</th>\n",
              "      <th>minimum_pass_rate</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>80%</td>\n",
              "      <td>66%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>80%</td>\n",
              "      <td>60%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  fail_count  pass_count pass_rate  \\\n",
              "0   robustness                   uppercase           2           8       80%   \n",
              "1   robustness                   lowercase           2           8       80%   \n",
              "2     fairness     min_gender_rouge1_score           1           2       67%   \n",
              "3     fairness     min_gender_rouge2_score           1           2       67%   \n",
              "4     fairness     min_gender_rougeL_score           1           2       67%   \n",
              "5     fairness  min_gender_rougeLsum_score           1           2       67%   \n",
              "6     fairness     max_gender_rouge1_score           2           1       33%   \n",
              "7     fairness     max_gender_rouge2_score           2           1       33%   \n",
              "8     fairness     max_gender_rougeL_score           2           1       33%   \n",
              "9     fairness  max_gender_rougeLsum_score           2           1       33%   \n",
              "10    accuracy       min_exact_match_score           1           0        0%   \n",
              "11    accuracy            min_rouge1_score           1           0        0%   \n",
              "12    accuracy            min_rougeL_score           1           0        0%   \n",
              "13    accuracy              min_bleu_score           1           0        0%   \n",
              "14    accuracy            min_rouge2_score           1           0        0%   \n",
              "15    accuracy         min_rougeLsum_score           1           0        0%   \n",
              "\n",
              "   minimum_pass_rate   pass  \n",
              "0                66%   True  \n",
              "1                60%   True  \n",
              "2                65%   True  \n",
              "3                65%   True  \n",
              "4                65%   True  \n",
              "5                65%   True  \n",
              "6                65%  False  \n",
              "7                65%  False  \n",
              "8                65%  False  \n",
              "9                65%  False  \n",
              "10               65%  False  \n",
              "11               65%  False  \n",
              "12               65%  False  \n",
              "13               65%  False  \n",
              "14               65%  False  \n",
              "15               65%  False  "
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.report()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IULGQtWAWp4L"
      },
      "source": [
        "## Bigbench-Causal-judgment Dataset Testing\n",
        "\n",
        "**Data Splits** \n",
        "- `test `  - Testing set from the Causal-judgment dataset, containing 190 question and answer examples. \n",
        "- `test-tiny`  -  Truncated version of Causal-judgment dataset which contains 50 question answer examples. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OoMGAn_FWpaP",
        "outputId": "cddff418-54ec-4bc7-91bc-e8118a8cae85"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Test Configuration : \n",
            " {\n",
            " \"model_parameters\": {\n",
            "  \"temperature\": 0.2,\n",
            "  \"max_tokens\": 64\n",
            " },\n",
            " \"tests\": {\n",
            "  \"defaults\": {\n",
            "   \"min_pass_rate\": 1.0\n",
            "  },\n",
            "  \"robustness\": {\n",
            "   \"add_typo\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   },\n",
            "   \"lowercase\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   }\n",
            "  }\n",
            " }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "harness = Harness(\n",
        "                  task=\"question-answering\", \n",
        "                  model={\"model\": \"gpt-3.5-turbo-instruct\",\"hub\":\"openai\"}, \n",
        "                  data={\"data_source\" :\"Bigbench\",\n",
        "                        \"subset\":\"Causal-judgment\",\n",
        "                        \"split\":\"test-tiny\"}\n",
        "                  )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "45-rhwhTXMWb",
        "outputId": "f9b1063c-ec3a-4891-b103-b946dfd3633f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'tests': {'defaults': {'min_pass_rate': 0.65},\n",
              "  'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
              "   'lowercase': {'min_pass_rate': 0.6}},\n",
              "  'fairness': {'min_gender_rouge1_score': {'min_score': 0.66},\n",
              "   'min_gender_rouge2_score': {'min_score': 0.6},\n",
              "   'min_gender_rougeL_score': {'min_score': 0.66},\n",
              "   'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
              "   'max_gender_rouge1_score': {'max_score': 0.66},\n",
              "   'max_gender_rouge2_score': {'max_score': 0.6},\n",
              "   'max_gender_rougeL_score': {'max_score': 0.66},\n",
              "   'max_gender_rougeLsum_score': {'max_score': 0.66}},\n",
              "  'accuracy': {'min_exact_match_score': {'min_score': 0.8},\n",
              "   'min_rouge1_score': {'min_score': 0.8},\n",
              "   'min_rougeL_score': {'min_score': 0.8},\n",
              "   'min_bleu_score': {'min_score': 0.8},\n",
              "   'min_rouge2_score': {'min_score': 0.8},\n",
              "   'min_rougeLsum_score': {'min_score': 0.8}}}}"
            ]
          },
          "execution_count": 40,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.configure(\n",
        "{\n",
        " 'tests': {'defaults': {'min_pass_rate': 0.65},\n",
        "           'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
        "                          'lowercase':{'min_pass_rate': 0.60},\n",
        "                        },\n",
        "           'fairness': {\n",
        "               'min_gender_rouge1_score': {'min_score': 0.66},\n",
        "               'min_gender_rouge2_score':{'min_score': 0.60},\n",
        "               'min_gender_rougeL_score': {'min_score': 0.66},\n",
        "               'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
        "               'max_gender_rouge1_score': {'max_score': 0.66},\n",
        "               'max_gender_rouge2_score':{'max_score': 0.60},\n",
        "               'max_gender_rougeL_score': {'max_score': 0.66},\n",
        "               'max_gender_rougeLsum_score': {'max_score': 0.66},\n",
        "                        },\n",
        "          'accuracy': {'min_exact_match_score': {'min_score': 0.80},\n",
        "                        'min_rouge1_score':{'min_score': 0.80},\n",
        "                        'min_rougeL_score':{'min_score': 0.80},\n",
        "                        'min_bleu_score':{'min_score': 0.80},\n",
        "                        'min_rouge2_score':{'min_score': 0.80},\n",
        "                        'min_rougeLsum_score':{'min_score': 0.80}\n",
        "\n",
        "                        }\n",
        "          }\n",
        " }\n",
        " )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dw85pgowGx8t"
      },
      "source": [
        "### Generating the Test Cases"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F2p1pXfoXzND",
        "outputId": "1043cf23-6f1b-4288-a810-ee8bbc67fc13"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "Generating testcases...: 100%|██████████| 3/3 [00:00<00:00, 5178.15it/s]\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "id": "vJZxMYyKX0Pe",
        "outputId": "9e61b93f-e51f-4537-c6d9-1463c55f3943"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-64fa49e9-5f89-4ee0-8dc4-1f8e798abe03\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The CEO of a company is sitting in his office ...</td>\n",
              "      <td>Did the CEO intentionally harm the environment?</td>\n",
              "      <td>THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...</td>\n",
              "      <td>DID THE CEO INTENTIONALLY HARM THE ENVIRONMENT?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The CEO of a company is sitting in his office ...</td>\n",
              "      <td>Did the CEO intentionally help the environment?</td>\n",
              "      <td>THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...</td>\n",
              "      <td>DID THE CEO INTENTIONALLY HELP THE ENVIRONMENT?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Lauren and Jane work for the same company. The...</td>\n",
              "      <td>Did Jane cause the computer to crash?</td>\n",
              "      <td>LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...</td>\n",
              "      <td>DID JANE CAUSE THE COMPUTER TO CRASH?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Lauren and Jane work for the same company. The...</td>\n",
              "      <td>Did Jane cause the computer to crash?</td>\n",
              "      <td>LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...</td>\n",
              "      <td>DID JANE CAUSE THE COMPUTER TO CRASH?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The receptionist in the philosophy department ...</td>\n",
              "      <td>Did Professor Smith cause the problem?</td>\n",
              "      <td>THE RECEPTIONIST IN THE PHILOSOPHY DEPARTMENT ...</td>\n",
              "      <td>DID PROFESSOR SMITH CAUSE THE PROBLEM?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>125</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>126</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>128</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>129</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>130 rows × 6 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-64fa49e9-5f89-4ee0-8dc4-1f8e798abe03')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-64fa49e9-5f89-4ee0-8dc4-1f8e798abe03 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-64fa49e9-5f89-4ee0-8dc4-1f8e798abe03');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9832fa5b-8b0a-4ac5-a468-b8f55292f242\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9832fa5b-8b0a-4ac5-a468-b8f55292f242')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9832fa5b-8b0a-4ac5-a468-b8f55292f242 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "       category            test_type  \\\n",
              "0    robustness            uppercase   \n",
              "1    robustness            uppercase   \n",
              "2    robustness            uppercase   \n",
              "3    robustness            uppercase   \n",
              "4    robustness            uppercase   \n",
              "..          ...                  ...   \n",
              "125    accuracy     min_rouge1_score   \n",
              "126    accuracy     min_rougeL_score   \n",
              "127    accuracy       min_bleu_score   \n",
              "128    accuracy     min_rouge2_score   \n",
              "129    accuracy  min_rougeLsum_score   \n",
              "\n",
              "                                      original_context  \\\n",
              "0    The CEO of a company is sitting in his office ...   \n",
              "1    The CEO of a company is sitting in his office ...   \n",
              "2    Lauren and Jane work for the same company. The...   \n",
              "3    Lauren and Jane work for the same company. The...   \n",
              "4    The receptionist in the philosophy department ...   \n",
              "..                                                 ...   \n",
              "125                                                  -   \n",
              "126                                                  -   \n",
              "127                                                  -   \n",
              "128                                                  -   \n",
              "129                                                  -   \n",
              "\n",
              "                                   original_question  \\\n",
              "0    Did the CEO intentionally harm the environment?   \n",
              "1    Did the CEO intentionally help the environment?   \n",
              "2              Did Jane cause the computer to crash?   \n",
              "3              Did Jane cause the computer to crash?   \n",
              "4             Did Professor Smith cause the problem?   \n",
              "..                                               ...   \n",
              "125                                                -   \n",
              "126                                                -   \n",
              "127                                                -   \n",
              "128                                                -   \n",
              "129                                                -   \n",
              "\n",
              "                                     perturbed_context  \\\n",
              "0    THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...   \n",
              "1    THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...   \n",
              "2    LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...   \n",
              "3    LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...   \n",
              "4    THE RECEPTIONIST IN THE PHILOSOPHY DEPARTMENT ...   \n",
              "..                                                 ...   \n",
              "125                                                  -   \n",
              "126                                                  -   \n",
              "127                                                  -   \n",
              "128                                                  -   \n",
              "129                                                  -   \n",
              "\n",
              "                                  perturbed_question  \n",
              "0    DID THE CEO INTENTIONALLY HARM THE ENVIRONMENT?  \n",
              "1    DID THE CEO INTENTIONALLY HELP THE ENVIRONMENT?  \n",
              "2              DID JANE CAUSE THE COMPUTER TO CRASH?  \n",
              "3              DID JANE CAUSE THE COMPUTER TO CRASH?  \n",
              "4             DID PROFESSOR SMITH CAUSE THE PROBLEM?  \n",
              "..                                               ...  \n",
              "125                                                -  \n",
              "126                                                -  \n",
              "127                                                -  \n",
              "128                                                -  \n",
              "129                                                -  \n",
              "\n",
              "[130 rows x 6 columns]"
            ]
          },
          "execution_count": 42,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.testcases()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zSgEmwr7G2Xl"
      },
      "source": [
        "### Running the tests"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "marZgGMEX2F1",
        "outputId": "2e8c56a0-cba1-43ca-f55a-90c57c671f13"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Running testcases... : 100%|██████████| 130/130 [01:26<00:00,  1.50it/s]\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.run()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "syaSCLsQIGiV"
      },
      "source": [
        "### Generated Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 704
        },
        "id": "ZoI8_JUBX4XC",
        "outputId": "a9ed94ef-f460-45f4-f710-f51da6adecc6"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-29a5941f-9af7-47d6-8322-f5d710f81d59\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The CEO of a company is sitting in his office ...</td>\n",
              "      <td>Did the CEO intentionally harm the environment?</td>\n",
              "      <td>THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...</td>\n",
              "      <td>DID THE CEO INTENTIONALLY HARM THE ENVIRONMENT?</td>\n",
              "      <td>No</td>\n",
              "      <td>No.</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The CEO of a company is sitting in his office ...</td>\n",
              "      <td>Did the CEO intentionally help the environment?</td>\n",
              "      <td>THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...</td>\n",
              "      <td>DID THE CEO INTENTIONALLY HELP THE ENVIRONMENT?</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Lauren and Jane work for the same company. The...</td>\n",
              "      <td>Did Jane cause the computer to crash?</td>\n",
              "      <td>LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...</td>\n",
              "      <td>DID JANE CAUSE THE COMPUTER TO CRASH?</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Lauren and Jane work for the same company. The...</td>\n",
              "      <td>Did Jane cause the computer to crash?</td>\n",
              "      <td>LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...</td>\n",
              "      <td>DID JANE CAUSE THE COMPUTER TO CRASH?</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The receptionist in the philosophy department ...</td>\n",
              "      <td>Did Professor Smith cause the problem?</td>\n",
              "      <td>THE RECEPTIONIST IN THE PHILOSOPHY DEPARTMENT ...</td>\n",
              "      <td>DID PROFESSOR SMITH CAUSE THE PROBLEM?</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>125</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.72</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>126</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.72</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>128</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>129</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.72</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>130 rows × 9 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-29a5941f-9af7-47d6-8322-f5d710f81d59')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-29a5941f-9af7-47d6-8322-f5d710f81d59 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-29a5941f-9af7-47d6-8322-f5d710f81d59');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-8f00e3b2-f822-4626-bf29-66d81e777e47\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-8f00e3b2-f822-4626-bf29-66d81e777e47')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-8f00e3b2-f822-4626-bf29-66d81e777e47 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "       category            test_type  \\\n",
              "0    robustness            uppercase   \n",
              "1    robustness            uppercase   \n",
              "2    robustness            uppercase   \n",
              "3    robustness            uppercase   \n",
              "4    robustness            uppercase   \n",
              "..          ...                  ...   \n",
              "125    accuracy     min_rouge1_score   \n",
              "126    accuracy     min_rougeL_score   \n",
              "127    accuracy       min_bleu_score   \n",
              "128    accuracy     min_rouge2_score   \n",
              "129    accuracy  min_rougeLsum_score   \n",
              "\n",
              "                                      original_context  \\\n",
              "0    The CEO of a company is sitting in his office ...   \n",
              "1    The CEO of a company is sitting in his office ...   \n",
              "2    Lauren and Jane work for the same company. The...   \n",
              "3    Lauren and Jane work for the same company. The...   \n",
              "4    The receptionist in the philosophy department ...   \n",
              "..                                                 ...   \n",
              "125                                                  -   \n",
              "126                                                  -   \n",
              "127                                                  -   \n",
              "128                                                  -   \n",
              "129                                                  -   \n",
              "\n",
              "                                   original_question  \\\n",
              "0    Did the CEO intentionally harm the environment?   \n",
              "1    Did the CEO intentionally help the environment?   \n",
              "2              Did Jane cause the computer to crash?   \n",
              "3              Did Jane cause the computer to crash?   \n",
              "4             Did Professor Smith cause the problem?   \n",
              "..                                               ...   \n",
              "125                                                -   \n",
              "126                                                -   \n",
              "127                                                -   \n",
              "128                                                -   \n",
              "129                                                -   \n",
              "\n",
              "                                     perturbed_context  \\\n",
              "0    THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...   \n",
              "1    THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...   \n",
              "2    LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...   \n",
              "3    LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...   \n",
              "4    THE RECEPTIONIST IN THE PHILOSOPHY DEPARTMENT ...   \n",
              "..                                                 ...   \n",
              "125                                                  -   \n",
              "126                                                  -   \n",
              "127                                                  -   \n",
              "128                                                  -   \n",
              "129                                                  -   \n",
              "\n",
              "                                  perturbed_question expected_result  \\\n",
              "0    DID THE CEO INTENTIONALLY HARM THE ENVIRONMENT?              No   \n",
              "1    DID THE CEO INTENTIONALLY HELP THE ENVIRONMENT?              No   \n",
              "2              DID JANE CAUSE THE COMPUTER TO CRASH?             Yes   \n",
              "3              DID JANE CAUSE THE COMPUTER TO CRASH?              No   \n",
              "4             DID PROFESSOR SMITH CAUSE THE PROBLEM?             Yes   \n",
              "..                                               ...             ...   \n",
              "125                                                -             0.8   \n",
              "126                                                -             0.8   \n",
              "127                                                -             0.8   \n",
              "128                                                -             0.8   \n",
              "129                                                -             0.8   \n",
              "\n",
              "    actual_result   pass  \n",
              "0             No.   True  \n",
              "1              No   True  \n",
              "2             Yes   True  \n",
              "3              No   True  \n",
              "4             Yes   True  \n",
              "..            ...    ...  \n",
              "125          0.72  False  \n",
              "126          0.72  False  \n",
              "127           0.0  False  \n",
              "128           0.0  False  \n",
              "129          0.72  False  \n",
              "\n",
              "[130 rows x 9 columns]"
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generated_results()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "GATFsQzLfEjB"
      },
      "outputs": [],
      "source": [
        "df=harness.generated_results()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "saUMk58mfCp6"
      },
      "source": [
        "### Generated Results for Robustness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 877
        },
        "id": "pBIhj5t0fCqe",
        "outputId": "4838e5c7-3db1-4252-ceb9-0836d2b4ff5e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-fca89cd9-13c2-46d5-91c0-9d0c70733a7b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The CEO of a company is sitting in his office ...</td>\n",
              "      <td>Did the CEO intentionally harm the environment?</td>\n",
              "      <td>THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...</td>\n",
              "      <td>DID THE CEO INTENTIONALLY HARM THE ENVIRONMENT?</td>\n",
              "      <td>No</td>\n",
              "      <td>No.</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The CEO of a company is sitting in his office ...</td>\n",
              "      <td>Did the CEO intentionally help the environment?</td>\n",
              "      <td>THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...</td>\n",
              "      <td>DID THE CEO INTENTIONALLY HELP THE ENVIRONMENT?</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Lauren and Jane work for the same company. The...</td>\n",
              "      <td>Did Jane cause the computer to crash?</td>\n",
              "      <td>LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...</td>\n",
              "      <td>DID JANE CAUSE THE COMPUTER TO CRASH?</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Lauren and Jane work for the same company. The...</td>\n",
              "      <td>Did Jane cause the computer to crash?</td>\n",
              "      <td>LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...</td>\n",
              "      <td>DID JANE CAUSE THE COMPUTER TO CRASH?</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The receptionist in the philosophy department ...</td>\n",
              "      <td>Did Professor Smith cause the problem?</td>\n",
              "      <td>THE RECEPTIONIST IN THE PHILOSOPHY DEPARTMENT ...</td>\n",
              "      <td>DID PROFESSOR SMITH CAUSE THE PROBLEM?</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>There is a man who gets paid for pumping water...</td>\n",
              "      <td>Did the man intentionally poison the inhabitants?</td>\n",
              "      <td>there is a man who gets paid for pumping water...</td>\n",
              "      <td>did the man intentionally poison the inhabitants?</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Frank T., had an ongoing dispute with his neig...</td>\n",
              "      <td>intentionally shoot his neighbor in the body?</td>\n",
              "      <td>frank t., had an ongoing dispute with his neig...</td>\n",
              "      <td>intentionally shoot his neighbor in the body?</td>\n",
              "      <td>No.</td>\n",
              "      <td>No.</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Frank T., had an ongoing dispute with his neig...</td>\n",
              "      <td>intentionally shoot his neighbor in the body?</td>\n",
              "      <td>frank t., had an ongoing dispute with his neig...</td>\n",
              "      <td>intentionally shoot his neighbor in the body?</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>George and his sister Lena reunite at their pa...</td>\n",
              "      <td>Did George hit the low point region intentiona...</td>\n",
              "      <td>george and his sister lena reunite at their pa...</td>\n",
              "      <td>did george hit the low point region intentiona...</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>George and his sister Lena reunite at their pa...</td>\n",
              "      <td>Did George hit the low point region intentiona...</td>\n",
              "      <td>george and his sister lena reunite at their pa...</td>\n",
              "      <td>did george hit the low point region intentiona...</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>100 rows × 9 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fca89cd9-13c2-46d5-91c0-9d0c70733a7b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-fca89cd9-13c2-46d5-91c0-9d0c70733a7b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-fca89cd9-13c2-46d5-91c0-9d0c70733a7b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b7d353f3-e69c-4e7e-85ec-c26da23831e7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b7d353f3-e69c-4e7e-85ec-c26da23831e7')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b7d353f3-e69c-4e7e-85ec-c26da23831e7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "      category  test_type                                   original_context  \\\n",
              "0   robustness  uppercase  The CEO of a company is sitting in his office ...   \n",
              "1   robustness  uppercase  The CEO of a company is sitting in his office ...   \n",
              "2   robustness  uppercase  Lauren and Jane work for the same company. The...   \n",
              "3   robustness  uppercase  Lauren and Jane work for the same company. The...   \n",
              "4   robustness  uppercase  The receptionist in the philosophy department ...   \n",
              "..         ...        ...                                                ...   \n",
              "95  robustness  lowercase  There is a man who gets paid for pumping water...   \n",
              "96  robustness  lowercase  Frank T., had an ongoing dispute with his neig...   \n",
              "97  robustness  lowercase  Frank T., had an ongoing dispute with his neig...   \n",
              "98  robustness  lowercase  George and his sister Lena reunite at their pa...   \n",
              "99  robustness  lowercase  George and his sister Lena reunite at their pa...   \n",
              "\n",
              "                                    original_question  \\\n",
              "0     Did the CEO intentionally harm the environment?   \n",
              "1     Did the CEO intentionally help the environment?   \n",
              "2               Did Jane cause the computer to crash?   \n",
              "3               Did Jane cause the computer to crash?   \n",
              "4              Did Professor Smith cause the problem?   \n",
              "..                                                ...   \n",
              "95  Did the man intentionally poison the inhabitants?   \n",
              "96      intentionally shoot his neighbor in the body?   \n",
              "97      intentionally shoot his neighbor in the body?   \n",
              "98  Did George hit the low point region intentiona...   \n",
              "99  Did George hit the low point region intentiona...   \n",
              "\n",
              "                                    perturbed_context  \\\n",
              "0   THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...   \n",
              "1   THE CEO OF A COMPANY IS SITTING IN HIS OFFICE ...   \n",
              "2   LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...   \n",
              "3   LAUREN AND JANE WORK FOR THE SAME COMPANY. THE...   \n",
              "4   THE RECEPTIONIST IN THE PHILOSOPHY DEPARTMENT ...   \n",
              "..                                                ...   \n",
              "95  there is a man who gets paid for pumping water...   \n",
              "96  frank t., had an ongoing dispute with his neig...   \n",
              "97  frank t., had an ongoing dispute with his neig...   \n",
              "98  george and his sister lena reunite at their pa...   \n",
              "99  george and his sister lena reunite at their pa...   \n",
              "\n",
              "                                   perturbed_question expected_result  \\\n",
              "0     DID THE CEO INTENTIONALLY HARM THE ENVIRONMENT?              No   \n",
              "1     DID THE CEO INTENTIONALLY HELP THE ENVIRONMENT?              No   \n",
              "2               DID JANE CAUSE THE COMPUTER TO CRASH?             Yes   \n",
              "3               DID JANE CAUSE THE COMPUTER TO CRASH?              No   \n",
              "4              DID PROFESSOR SMITH CAUSE THE PROBLEM?             Yes   \n",
              "..                                                ...             ...   \n",
              "95  did the man intentionally poison the inhabitants?              No   \n",
              "96      intentionally shoot his neighbor in the body?             No.   \n",
              "97      intentionally shoot his neighbor in the body?             Yes   \n",
              "98  did george hit the low point region intentiona...             Yes   \n",
              "99  did george hit the low point region intentiona...              No   \n",
              "\n",
              "   actual_result  pass  \n",
              "0            No.  True  \n",
              "1             No  True  \n",
              "2            Yes  True  \n",
              "3             No  True  \n",
              "4            Yes  True  \n",
              "..           ...   ...  \n",
              "95            No  True  \n",
              "96           No.  True  \n",
              "97           Yes  True  \n",
              "98           Yes  True  \n",
              "99            No  True  \n",
              "\n",
              "[100 rows x 9 columns]"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"robustness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rk0HFPY4fCqf"
      },
      "source": [
        "### Generated Results for fairness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 822
        },
        "id": "_G9jmAvZfCqg",
        "outputId": "a001e64a-b56d-421f-b764-2d49d4b9057d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-407ac88d-1544-41eb-ac7b-6cc6ed25c9ab\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>101</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.875</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>102</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>104</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>105</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>106</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.875</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>108</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>109</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.875</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>111</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>112</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>113</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.875</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>114</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>115</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>116</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>117</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>119</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.875</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>120</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>121</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>122</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.875</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>123</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-407ac88d-1544-41eb-ac7b-6cc6ed25c9ab')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-407ac88d-1544-41eb-ac7b-6cc6ed25c9ab button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-407ac88d-1544-41eb-ac7b-6cc6ed25c9ab');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-8f16ac71-c6c7-4d8a-8077-629b238c9060\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-8f16ac71-c6c7-4d8a-8077-629b238c9060')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-8f16ac71-c6c7-4d8a-8077-629b238c9060 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "     category                   test_type original_context original_question  \\\n",
              "100  fairness     min_gender_rouge1_score                -              male   \n",
              "101  fairness     min_gender_rouge1_score                -            female   \n",
              "102  fairness     min_gender_rouge1_score                -           unknown   \n",
              "103  fairness     min_gender_rouge2_score                -              male   \n",
              "104  fairness     min_gender_rouge2_score                -            female   \n",
              "105  fairness     min_gender_rouge2_score                -           unknown   \n",
              "106  fairness     min_gender_rougeL_score                -              male   \n",
              "107  fairness     min_gender_rougeL_score                -            female   \n",
              "108  fairness     min_gender_rougeL_score                -           unknown   \n",
              "109  fairness  min_gender_rougeLsum_score                -              male   \n",
              "110  fairness  min_gender_rougeLsum_score                -            female   \n",
              "111  fairness  min_gender_rougeLsum_score                -           unknown   \n",
              "112  fairness     max_gender_rouge1_score                -              male   \n",
              "113  fairness     max_gender_rouge1_score                -            female   \n",
              "114  fairness     max_gender_rouge1_score                -           unknown   \n",
              "115  fairness     max_gender_rouge2_score                -              male   \n",
              "116  fairness     max_gender_rouge2_score                -            female   \n",
              "117  fairness     max_gender_rouge2_score                -           unknown   \n",
              "118  fairness     max_gender_rougeL_score                -              male   \n",
              "119  fairness     max_gender_rougeL_score                -            female   \n",
              "120  fairness     max_gender_rougeL_score                -           unknown   \n",
              "121  fairness  max_gender_rougeLsum_score                -              male   \n",
              "122  fairness  max_gender_rougeLsum_score                -            female   \n",
              "123  fairness  max_gender_rougeLsum_score                -           unknown   \n",
              "\n",
              "    perturbed_context perturbed_question expected_result actual_result   pass  \n",
              "100                 -                  -            0.66      0.666667   True  \n",
              "101                 -                  -            0.66         0.875   True  \n",
              "102                 -                  -            0.66           1.0   True  \n",
              "103                 -                  -             0.6           0.0  False  \n",
              "104                 -                  -             0.6           0.0  False  \n",
              "105                 -                  -             0.6           0.0  False  \n",
              "106                 -                  -            0.66      0.666667   True  \n",
              "107                 -                  -            0.66         0.875   True  \n",
              "108                 -                  -            0.66           1.0   True  \n",
              "109                 -                  -            0.66      0.666667   True  \n",
              "110                 -                  -            0.66         0.875   True  \n",
              "111                 -                  -            0.66           1.0   True  \n",
              "112                 -                  -            0.66      0.666667  False  \n",
              "113                 -                  -            0.66         0.875  False  \n",
              "114                 -                  -            0.66           1.0  False  \n",
              "115                 -                  -             0.6           0.0   True  \n",
              "116                 -                  -             0.6           0.0   True  \n",
              "117                 -                  -             0.6           0.0   True  \n",
              "118                 -                  -            0.66      0.666667  False  \n",
              "119                 -                  -            0.66         0.875  False  \n",
              "120                 -                  -            0.66           1.0  False  \n",
              "121                 -                  -            0.66      0.666667  False  \n",
              "122                 -                  -            0.66         0.875  False  \n",
              "123                 -                  -            0.66           1.0  False  "
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"fairness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yzxbzh5WfCqh"
      },
      "source": [
        "### Generated Results for accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "BcdBdqWMfCqh",
        "outputId": "ec345c60-3acf-444e-98fb-3e98194343cc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-c59e4283-b059-4dca-a2f7-a9cdeb626c83\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>124</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.58</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>125</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.72</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>126</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.72</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>128</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>129</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.72</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c59e4283-b059-4dca-a2f7-a9cdeb626c83')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c59e4283-b059-4dca-a2f7-a9cdeb626c83 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c59e4283-b059-4dca-a2f7-a9cdeb626c83');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-298ffe72-e4d1-40c3-88fc-a6b6cfd35b05\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-298ffe72-e4d1-40c3-88fc-a6b6cfd35b05')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-298ffe72-e4d1-40c3-88fc-a6b6cfd35b05 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "     category              test_type original_context original_question  \\\n",
              "124  accuracy  min_exact_match_score                -                 -   \n",
              "125  accuracy       min_rouge1_score                -                 -   \n",
              "126  accuracy       min_rougeL_score                -                 -   \n",
              "127  accuracy         min_bleu_score                -                 -   \n",
              "128  accuracy       min_rouge2_score                -                 -   \n",
              "129  accuracy    min_rougeLsum_score                -                 -   \n",
              "\n",
              "    perturbed_context perturbed_question expected_result actual_result   pass  \n",
              "124                 -                  -             0.8          0.58  False  \n",
              "125                 -                  -             0.8          0.72  False  \n",
              "126                 -                  -             0.8          0.72  False  \n",
              "127                 -                  -             0.8           0.0  False  \n",
              "128                 -                  -             0.8           0.0  False  \n",
              "129                 -                  -             0.8          0.72  False  "
            ]
          },
          "execution_count": 48,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"accuracy\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o39sXReLG7K9"
      },
      "source": [
        "### Final Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 551
        },
        "id": "AiyJ7SyJYC9V",
        "outputId": "96bc96d7-6226-4be5-8c5c-c70ae11785f0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-bb897678-75e3-471c-a0ae-19632ea11a2d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>fail_count</th>\n",
              "      <th>pass_count</th>\n",
              "      <th>pass_rate</th>\n",
              "      <th>minimum_pass_rate</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>4</td>\n",
              "      <td>46</td>\n",
              "      <td>92%</td>\n",
              "      <td>66%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>0</td>\n",
              "      <td>50</td>\n",
              "      <td>100%</td>\n",
              "      <td>60%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bb897678-75e3-471c-a0ae-19632ea11a2d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bb897678-75e3-471c-a0ae-19632ea11a2d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bb897678-75e3-471c-a0ae-19632ea11a2d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-08d15935-d175-4634-9159-99746d72c6f1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-08d15935-d175-4634-9159-99746d72c6f1')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const charts = await google.colab.kernel.invokeFunction(\n",
              "          'suggestCharts', [key], {});\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-08d15935-d175-4634-9159-99746d72c6f1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "      category                   test_type  fail_count  pass_count pass_rate  \\\n",
              "0   robustness                   uppercase           4          46       92%   \n",
              "1   robustness                   lowercase           0          50      100%   \n",
              "2     fairness     min_gender_rouge1_score           0           3      100%   \n",
              "3     fairness     min_gender_rouge2_score           3           0        0%   \n",
              "4     fairness     min_gender_rougeL_score           0           3      100%   \n",
              "5     fairness  min_gender_rougeLsum_score           0           3      100%   \n",
              "6     fairness     max_gender_rouge1_score           3           0        0%   \n",
              "7     fairness     max_gender_rouge2_score           0           3      100%   \n",
              "8     fairness     max_gender_rougeL_score           3           0        0%   \n",
              "9     fairness  max_gender_rougeLsum_score           3           0        0%   \n",
              "10    accuracy       min_exact_match_score           1           0        0%   \n",
              "11    accuracy            min_rouge1_score           1           0        0%   \n",
              "12    accuracy            min_rougeL_score           1           0        0%   \n",
              "13    accuracy              min_bleu_score           1           0        0%   \n",
              "14    accuracy            min_rouge2_score           1           0        0%   \n",
              "15    accuracy         min_rougeLsum_score           1           0        0%   \n",
              "\n",
              "   minimum_pass_rate   pass  \n",
              "0                66%   True  \n",
              "1                60%   True  \n",
              "2                65%   True  \n",
              "3                65%  False  \n",
              "4                65%   True  \n",
              "5                65%   True  \n",
              "6                65%  False  \n",
              "7                65%   True  \n",
              "8                65%  False  \n",
              "9                65%  False  \n",
              "10               65%  False  \n",
              "11               65%  False  \n",
              "12               65%  False  \n",
              "13               65%  False  \n",
              "14               65%  False  \n",
              "15               65%  False  "
            ]
          },
          "execution_count": 49,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.report()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0jSkCQudYh3F"
      },
      "source": [
        "## Bigbench-DisflQA Dataset Testing\n",
        "\n",
        "Data Splits\n",
        "-  `test` - Testing set from the DisfilQA subset, containing 1k question and answer examples. \n",
        "- `test-tiny ` - Truncated version of DisfilQA subset which contains 50 "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qG3UX5c-YgJn",
        "outputId": "305a542d-069e-4960-af17-72962b04a4e2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Test Configuration : \n",
            " {\n",
            " \"model_parameters\": {\n",
            "  \"max_tokens\": 64\n",
            " },\n",
            " \"tests\": {\n",
            "  \"defaults\": {\n",
            "   \"min_pass_rate\": 1.0\n",
            "  },\n",
            "  \"robustness\": {\n",
            "   \"add_typo\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   },\n",
            "   \"lowercase\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   }\n",
            "  }\n",
            " }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "harness = Harness(\n",
        "                  task=\"question-answering\", \n",
        "                  model={\"model\": \"gpt-3.5-turbo-instruct\",\"hub\":\"openai\"}, \n",
        "                  data={\"data_source\" :\"Bigbench\",\n",
        "                        \"subset\":\"DisflQA\",\n",
        "                        \"split\":\"test-tiny\"}\n",
        "                  )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KuLxNXwXYl2z",
        "outputId": "b1b4d57a-ff6c-4f64-e780-ff377eef865a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'tests': {'defaults': {'min_pass_rate': 0.65},\n",
              "  'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
              "   'lowercase': {'min_pass_rate': 0.6}},\n",
              "  'fairness': {'min_gender_rouge1_score': {'min_score': 0.66},\n",
              "   'min_gender_rouge2_score': {'min_score': 0.6},\n",
              "   'min_gender_rougeL_score': {'min_score': 0.66},\n",
              "   'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
              "   'max_gender_rouge1_score': {'max_score': 0.66},\n",
              "   'max_gender_rouge2_score': {'max_score': 0.6},\n",
              "   'max_gender_rougeL_score': {'max_score': 0.66},\n",
              "   'max_gender_rougeLsum_score': {'max_score': 0.66}},\n",
              "  'accuracy': {'min_exact_match_score': {'min_score': 0.8},\n",
              "   'min_rouge1_score': {'min_score': 0.8},\n",
              "   'min_rougeL_score': {'min_score': 0.8},\n",
              "   'min_bleu_score': {'min_score': 0.8},\n",
              "   'min_rouge2_score': {'min_score': 0.8},\n",
              "   'min_rougeLsum_score': {'min_score': 0.8}}}}"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.configure(\n",
        "{\n",
        " 'tests': {'defaults': {'min_pass_rate': 0.65},\n",
        "           'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
        "                          'lowercase':{'min_pass_rate': 0.60},\n",
        "                        },\n",
        "           'fairness': {\n",
        "               'min_gender_rouge1_score': {'min_score': 0.66},\n",
        "               'min_gender_rouge2_score':{'min_score': 0.60},\n",
        "               'min_gender_rougeL_score': {'min_score': 0.66},\n",
        "               'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
        "               'max_gender_rouge1_score': {'max_score': 0.66},\n",
        "               'max_gender_rouge2_score':{'max_score': 0.60},\n",
        "               'max_gender_rougeL_score': {'max_score': 0.66},\n",
        "               'max_gender_rougeLsum_score': {'max_score': 0.66},\n",
        "                        },\n",
        "          'accuracy': {'min_exact_match_score': {'min_score': 0.80},\n",
        "                        'min_rouge1_score':{'min_score': 0.80},\n",
        "                        'min_rougeL_score':{'min_score': 0.80},\n",
        "                        'min_bleu_score':{'min_score': 0.80},\n",
        "                        'min_rouge2_score':{'min_score': 0.80},\n",
        "                        'min_rougeLsum_score':{'min_score': 0.80}\n",
        "\n",
        "                        }\n",
        "          }\n",
        " }\n",
        " )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hd6BEnBtHyME"
      },
      "source": [
        "### Generating the test cases."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [],
      "source": [
        "harness.data = harness.data[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4_wMTSmbYqTa",
        "outputId": "019a0bc1-bc04-42b6-af12-46665db54e6b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generating testcases...: 100%|██████████| 3/3 [00:00<00:00, 2992.37it/s]\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 614
        },
        "id": "W28l71dScgG0",
        "outputId": "22f65e2b-733d-49ea-d3c8-9ee0e1d482ce"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The common allotrope of elemental oxygen on Ea...</td>\n",
              "      <td>What part the composition of the Earth ' s bio...</td>\n",
              "      <td>THE COMMON ALLOTROPE OF ELEMENTAL OXYGEN ON EA...</td>\n",
              "      <td>WHAT PART THE COMPOSITION OF THE EARTH ' S BIO...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>In addition to identifying rocks in the field ...</td>\n",
              "      <td>What do petrologists use rock samples or rathe...</td>\n",
              "      <td>IN ADDITION TO IDENTIFYING ROCKS IN THE FIELD ...</td>\n",
              "      <td>WHAT DO PETROLOGISTS USE ROCK SAMPLES OR RATHE...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The four - year , full - time undergraduate pr...</td>\n",
              "      <td>Post 2008 undergraduate students are required ...</td>\n",
              "      <td>THE FOUR - YEAR , FULL - TIME UNDERGRADUATE PR...</td>\n",
              "      <td>POST 2008 UNDERGRADUATE STUDENTS ARE REQUIRED ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Some Normans joined Turkish forces to aid in t...</td>\n",
              "      <td>How was or no make that what was the Norman ca...</td>\n",
              "      <td>SOME NORMANS JOINED TURKISH FORCES TO AID IN T...</td>\n",
              "      <td>HOW WAS OR NO MAKE THAT WHAT WAS THE NORMAN CA...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Current faculty include the anthropologist Mar...</td>\n",
              "      <td>Who is the current , oh no , what Shakespeare ...</td>\n",
              "      <td>CURRENT FACULTY INCLUDE THE ANTHROPOLOGIST MAR...</td>\n",
              "      <td>WHO IS THE CURRENT , OH NO , WHAT SHAKESPEARE ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The University of Chicago has been the site of...</td>\n",
              "      <td>What role in academic or I mean in economics d...</td>\n",
              "      <td>THE UNIVERSITY OF CHICAGO HAS BEEN THE SITE OF...</td>\n",
              "      <td>WHAT ROLE IN ACADEMIC OR I MEAN IN ECONOMICS D...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>As interesting examples of expositions the mos...</td>\n",
              "      <td>What is Warsaw ' s National Museum most known ...</td>\n",
              "      <td>AS INTERESTING EXAMPLES OF EXPOSITIONS THE MOS...</td>\n",
              "      <td>WHAT IS WARSAW ' S NATIONAL MUSEUM MOST KNOWN ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Since ~ 3000 yr BP ( = years Before Present ) ...</td>\n",
              "      <td>What has increased salt uhh rather sediment an...</td>\n",
              "      <td>SINCE ~ 3000 YR BP ( = YEARS BEFORE PRESENT ) ...</td>\n",
              "      <td>WHAT HAS INCREASED SALT UHH RATHER SEDIMENT AN...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The zeta function is closely related to prime ...</td>\n",
              "      <td>What does it mean when a prime number no when ...</td>\n",
              "      <td>THE ZETA FUNCTION IS CLOSELY RELATED TO PRIME ...</td>\n",
              "      <td>WHAT DOES IT MEAN WHEN A PRIME NUMBER NO WHEN ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Large - scale construction requires collaborat...</td>\n",
              "      <td>How does the no what does a construction engin...</td>\n",
              "      <td>LARGE - SCALE CONSTRUCTION REQUIRES COLLABORAT...</td>\n",
              "      <td>HOW DOES THE NO WHAT DOES A CONSTRUCTION ENGIN...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The common allotrope of elemental oxygen on Ea...</td>\n",
              "      <td>What part the composition of the Earth ' s bio...</td>\n",
              "      <td>the common allotrope of elemental oxygen on ea...</td>\n",
              "      <td>what part the composition of the earth ' s bio...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>In addition to identifying rocks in the field ...</td>\n",
              "      <td>What do petrologists use rock samples or rathe...</td>\n",
              "      <td>in addition to identifying rocks in the field ...</td>\n",
              "      <td>what do petrologists use rock samples or rathe...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The four - year , full - time undergraduate pr...</td>\n",
              "      <td>Post 2008 undergraduate students are required ...</td>\n",
              "      <td>the four - year , full - time undergraduate pr...</td>\n",
              "      <td>post 2008 undergraduate students are required ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Some Normans joined Turkish forces to aid in t...</td>\n",
              "      <td>How was or no make that what was the Norman ca...</td>\n",
              "      <td>some normans joined turkish forces to aid in t...</td>\n",
              "      <td>how was or no make that what was the norman ca...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Current faculty include the anthropologist Mar...</td>\n",
              "      <td>Who is the current , oh no , what Shakespeare ...</td>\n",
              "      <td>current faculty include the anthropologist mar...</td>\n",
              "      <td>who is the current , oh no , what shakespeare ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The University of Chicago has been the site of...</td>\n",
              "      <td>What role in academic or I mean in economics d...</td>\n",
              "      <td>the university of chicago has been the site of...</td>\n",
              "      <td>what role in academic or i mean in economics d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>As interesting examples of expositions the mos...</td>\n",
              "      <td>What is Warsaw ' s National Museum most known ...</td>\n",
              "      <td>as interesting examples of expositions the mos...</td>\n",
              "      <td>what is warsaw ' s national museum most known ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Since ~ 3000 yr BP ( = years Before Present ) ...</td>\n",
              "      <td>What has increased salt uhh rather sediment an...</td>\n",
              "      <td>since ~ 3000 yr bp ( = years before present ) ...</td>\n",
              "      <td>what has increased salt uhh rather sediment an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The zeta function is closely related to prime ...</td>\n",
              "      <td>What does it mean when a prime number no when ...</td>\n",
              "      <td>the zeta function is closely related to prime ...</td>\n",
              "      <td>what does it mean when a prime number no when ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Large - scale construction requires collaborat...</td>\n",
              "      <td>How does the no what does a construction engin...</td>\n",
              "      <td>large - scale construction requires collaborat...</td>\n",
              "      <td>how does the no what does a construction engin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  \\\n",
              "0   robustness                   uppercase   \n",
              "1   robustness                   uppercase   \n",
              "2   robustness                   uppercase   \n",
              "3   robustness                   uppercase   \n",
              "4   robustness                   uppercase   \n",
              "5   robustness                   uppercase   \n",
              "6   robustness                   uppercase   \n",
              "7   robustness                   uppercase   \n",
              "8   robustness                   uppercase   \n",
              "9   robustness                   uppercase   \n",
              "10  robustness                   lowercase   \n",
              "11  robustness                   lowercase   \n",
              "12  robustness                   lowercase   \n",
              "13  robustness                   lowercase   \n",
              "14  robustness                   lowercase   \n",
              "15  robustness                   lowercase   \n",
              "16  robustness                   lowercase   \n",
              "17  robustness                   lowercase   \n",
              "18  robustness                   lowercase   \n",
              "19  robustness                   lowercase   \n",
              "20    fairness     min_gender_rouge1_score   \n",
              "21    fairness     min_gender_rouge1_score   \n",
              "22    fairness     min_gender_rouge1_score   \n",
              "23    fairness     min_gender_rouge2_score   \n",
              "24    fairness     min_gender_rouge2_score   \n",
              "25    fairness     min_gender_rouge2_score   \n",
              "26    fairness     min_gender_rougeL_score   \n",
              "27    fairness     min_gender_rougeL_score   \n",
              "28    fairness     min_gender_rougeL_score   \n",
              "29    fairness  min_gender_rougeLsum_score   \n",
              "30    fairness  min_gender_rougeLsum_score   \n",
              "31    fairness  min_gender_rougeLsum_score   \n",
              "32    fairness     max_gender_rouge1_score   \n",
              "33    fairness     max_gender_rouge1_score   \n",
              "34    fairness     max_gender_rouge1_score   \n",
              "35    fairness     max_gender_rouge2_score   \n",
              "36    fairness     max_gender_rouge2_score   \n",
              "37    fairness     max_gender_rouge2_score   \n",
              "38    fairness     max_gender_rougeL_score   \n",
              "39    fairness     max_gender_rougeL_score   \n",
              "40    fairness     max_gender_rougeL_score   \n",
              "41    fairness  max_gender_rougeLsum_score   \n",
              "42    fairness  max_gender_rougeLsum_score   \n",
              "43    fairness  max_gender_rougeLsum_score   \n",
              "44    accuracy       min_exact_match_score   \n",
              "45    accuracy            min_rouge1_score   \n",
              "46    accuracy            min_rougeL_score   \n",
              "47    accuracy              min_bleu_score   \n",
              "48    accuracy            min_rouge2_score   \n",
              "49    accuracy         min_rougeLsum_score   \n",
              "\n",
              "                                     original_context  \\\n",
              "0   The common allotrope of elemental oxygen on Ea...   \n",
              "1   In addition to identifying rocks in the field ...   \n",
              "2   The four - year , full - time undergraduate pr...   \n",
              "3   Some Normans joined Turkish forces to aid in t...   \n",
              "4   Current faculty include the anthropologist Mar...   \n",
              "5   The University of Chicago has been the site of...   \n",
              "6   As interesting examples of expositions the mos...   \n",
              "7   Since ~ 3000 yr BP ( = years Before Present ) ...   \n",
              "8   The zeta function is closely related to prime ...   \n",
              "9   Large - scale construction requires collaborat...   \n",
              "10  The common allotrope of elemental oxygen on Ea...   \n",
              "11  In addition to identifying rocks in the field ...   \n",
              "12  The four - year , full - time undergraduate pr...   \n",
              "13  Some Normans joined Turkish forces to aid in t...   \n",
              "14  Current faculty include the anthropologist Mar...   \n",
              "15  The University of Chicago has been the site of...   \n",
              "16  As interesting examples of expositions the mos...   \n",
              "17  Since ~ 3000 yr BP ( = years Before Present ) ...   \n",
              "18  The zeta function is closely related to prime ...   \n",
              "19  Large - scale construction requires collaborat...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                    original_question  \\\n",
              "0   What part the composition of the Earth ' s bio...   \n",
              "1   What do petrologists use rock samples or rathe...   \n",
              "2   Post 2008 undergraduate students are required ...   \n",
              "3   How was or no make that what was the Norman ca...   \n",
              "4   Who is the current , oh no , what Shakespeare ...   \n",
              "5   What role in academic or I mean in economics d...   \n",
              "6   What is Warsaw ' s National Museum most known ...   \n",
              "7   What has increased salt uhh rather sediment an...   \n",
              "8   What does it mean when a prime number no when ...   \n",
              "9   How does the no what does a construction engin...   \n",
              "10  What part the composition of the Earth ' s bio...   \n",
              "11  What do petrologists use rock samples or rathe...   \n",
              "12  Post 2008 undergraduate students are required ...   \n",
              "13  How was or no make that what was the Norman ca...   \n",
              "14  Who is the current , oh no , what Shakespeare ...   \n",
              "15  What role in academic or I mean in economics d...   \n",
              "16  What is Warsaw ' s National Museum most known ...   \n",
              "17  What has increased salt uhh rather sediment an...   \n",
              "18  What does it mean when a prime number no when ...   \n",
              "19  How does the no what does a construction engin...   \n",
              "20                                               male   \n",
              "21                                             female   \n",
              "22                                            unknown   \n",
              "23                                               male   \n",
              "24                                             female   \n",
              "25                                            unknown   \n",
              "26                                               male   \n",
              "27                                             female   \n",
              "28                                            unknown   \n",
              "29                                               male   \n",
              "30                                             female   \n",
              "31                                            unknown   \n",
              "32                                               male   \n",
              "33                                             female   \n",
              "34                                            unknown   \n",
              "35                                               male   \n",
              "36                                             female   \n",
              "37                                            unknown   \n",
              "38                                               male   \n",
              "39                                             female   \n",
              "40                                            unknown   \n",
              "41                                               male   \n",
              "42                                             female   \n",
              "43                                            unknown   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                    perturbed_context  \\\n",
              "0   THE COMMON ALLOTROPE OF ELEMENTAL OXYGEN ON EA...   \n",
              "1   IN ADDITION TO IDENTIFYING ROCKS IN THE FIELD ...   \n",
              "2   THE FOUR - YEAR , FULL - TIME UNDERGRADUATE PR...   \n",
              "3   SOME NORMANS JOINED TURKISH FORCES TO AID IN T...   \n",
              "4   CURRENT FACULTY INCLUDE THE ANTHROPOLOGIST MAR...   \n",
              "5   THE UNIVERSITY OF CHICAGO HAS BEEN THE SITE OF...   \n",
              "6   AS INTERESTING EXAMPLES OF EXPOSITIONS THE MOS...   \n",
              "7   SINCE ~ 3000 YR BP ( = YEARS BEFORE PRESENT ) ...   \n",
              "8   THE ZETA FUNCTION IS CLOSELY RELATED TO PRIME ...   \n",
              "9   LARGE - SCALE CONSTRUCTION REQUIRES COLLABORAT...   \n",
              "10  the common allotrope of elemental oxygen on ea...   \n",
              "11  in addition to identifying rocks in the field ...   \n",
              "12  the four - year , full - time undergraduate pr...   \n",
              "13  some normans joined turkish forces to aid in t...   \n",
              "14  current faculty include the anthropologist mar...   \n",
              "15  the university of chicago has been the site of...   \n",
              "16  as interesting examples of expositions the mos...   \n",
              "17  since ~ 3000 yr bp ( = years before present ) ...   \n",
              "18  the zeta function is closely related to prime ...   \n",
              "19  large - scale construction requires collaborat...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                   perturbed_question  \n",
              "0   WHAT PART THE COMPOSITION OF THE EARTH ' S BIO...  \n",
              "1   WHAT DO PETROLOGISTS USE ROCK SAMPLES OR RATHE...  \n",
              "2   POST 2008 UNDERGRADUATE STUDENTS ARE REQUIRED ...  \n",
              "3   HOW WAS OR NO MAKE THAT WHAT WAS THE NORMAN CA...  \n",
              "4   WHO IS THE CURRENT , OH NO , WHAT SHAKESPEARE ...  \n",
              "5   WHAT ROLE IN ACADEMIC OR I MEAN IN ECONOMICS D...  \n",
              "6   WHAT IS WARSAW ' S NATIONAL MUSEUM MOST KNOWN ...  \n",
              "7   WHAT HAS INCREASED SALT UHH RATHER SEDIMENT AN...  \n",
              "8   WHAT DOES IT MEAN WHEN A PRIME NUMBER NO WHEN ...  \n",
              "9   HOW DOES THE NO WHAT DOES A CONSTRUCTION ENGIN...  \n",
              "10  what part the composition of the earth ' s bio...  \n",
              "11  what do petrologists use rock samples or rathe...  \n",
              "12  post 2008 undergraduate students are required ...  \n",
              "13  how was or no make that what was the norman ca...  \n",
              "14  who is the current , oh no , what shakespeare ...  \n",
              "15  what role in academic or i mean in economics d...  \n",
              "16  what is warsaw ' s national museum most known ...  \n",
              "17  what has increased salt uhh rather sediment an...  \n",
              "18  what does it mean when a prime number no when ...  \n",
              "19  how does the no what does a construction engin...  \n",
              "20                                                  -  \n",
              "21                                                  -  \n",
              "22                                                  -  \n",
              "23                                                  -  \n",
              "24                                                  -  \n",
              "25                                                  -  \n",
              "26                                                  -  \n",
              "27                                                  -  \n",
              "28                                                  -  \n",
              "29                                                  -  \n",
              "30                                                  -  \n",
              "31                                                  -  \n",
              "32                                                  -  \n",
              "33                                                  -  \n",
              "34                                                  -  \n",
              "35                                                  -  \n",
              "36                                                  -  \n",
              "37                                                  -  \n",
              "38                                                  -  \n",
              "39                                                  -  \n",
              "40                                                  -  \n",
              "41                                                  -  \n",
              "42                                                  -  \n",
              "43                                                  -  \n",
              "44                                                  -  \n",
              "45                                                  -  \n",
              "46                                                  -  \n",
              "47                                                  -  \n",
              "48                                                  -  \n",
              "49                                                  -  "
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.testcases()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UsbsuknXH0ue"
      },
      "source": [
        "### Running the tests"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PxeBTKR9chtd",
        "outputId": "fb70a2b0-19c1-44d7-830f-e25e7a71f733"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Running testcases... : 100%|██████████| 50/50 [02:30<00:00,  3.69s/it]"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.run()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LDYWRg6DIC4B"
      },
      "source": [
        "### Generated Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 738
        },
        "id": "xzjd-oQvcji8",
        "outputId": "d962e366-5e09-4c93-90d3-48d69c4321a0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The common allotrope of elemental oxygen on Ea...</td>\n",
              "      <td>What part the composition of the Earth ' s bio...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE COMMON ALLOTROPE OF ELEMENTAL OXYGEN ON EA...</td>\n",
              "      <td>WHAT PART THE COMPOSITION OF THE EARTH ' S BIO...</td>\n",
              "      <td>The Earth's atmosphere is comprised of dioxyg...</td>\n",
              "      <td>The common allotrope of elemental oxygen on E...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>In addition to identifying rocks in the field ...</td>\n",
              "      <td>What do petrologists use rock samples or rathe...</td>\n",
              "      <td>-</td>\n",
              "      <td>IN ADDITION TO IDENTIFYING ROCKS IN THE FIELD ...</td>\n",
              "      <td>WHAT DO PETROLOGISTS USE ROCK SAMPLES OR RATHE...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The four - year , full - time undergraduate pr...</td>\n",
              "      <td>Post 2008 undergraduate students are required ...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE FOUR - YEAR , FULL - TIME UNDERGRADUATE PR...</td>\n",
              "      <td>POST 2008 UNDERGRADUATE STUDENTS ARE REQUIRED ...</td>\n",
              "      <td>Since 2008, undergraduate students have been ...</td>\n",
              "      <td>Post 2008 undergraduate students are required...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Some Normans joined Turkish forces to aid in t...</td>\n",
              "      <td>How was or no make that what was the Norman ca...</td>\n",
              "      <td>-</td>\n",
              "      <td>SOME NORMANS JOINED TURKISH FORCES TO AID IN T...</td>\n",
              "      <td>HOW WAS OR NO MAKE THAT WHAT WAS THE NORMAN CA...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Current faculty include the anthropologist Mar...</td>\n",
              "      <td>Who is the current , oh no , what Shakespeare ...</td>\n",
              "      <td>-</td>\n",
              "      <td>CURRENT FACULTY INCLUDE THE ANTHROPOLOGIST MAR...</td>\n",
              "      <td>WHO IS THE CURRENT , OH NO , WHAT SHAKESPEARE ...</td>\n",
              "      <td>Shakespeare scholar David Bevington is curren...</td>\n",
              "      <td>David Bevington.</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The University of Chicago has been the site of...</td>\n",
              "      <td>What role in academic or I mean in economics d...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE UNIVERSITY OF CHICAGO HAS BEEN THE SITE OF...</td>\n",
              "      <td>WHAT ROLE IN ACADEMIC OR I MEAN IN ECONOMICS D...</td>\n",
              "      <td>The University of Chicago played an important...</td>\n",
              "      <td>The University of Chicago played an important...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>As interesting examples of expositions the mos...</td>\n",
              "      <td>What is Warsaw ' s National Museum most known ...</td>\n",
              "      <td>-</td>\n",
              "      <td>AS INTERESTING EXAMPLES OF EXPOSITIONS THE MOS...</td>\n",
              "      <td>WHAT IS WARSAW ' S NATIONAL MUSEUM MOST KNOWN ...</td>\n",
              "      <td>The National Museum in Warsaw is known for it...</td>\n",
              "      <td>The National Museum in Warsaw is most known f...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Since ~ 3000 yr BP ( = years Before Present ) ...</td>\n",
              "      <td>What has increased salt uhh rather sediment an...</td>\n",
              "      <td>-</td>\n",
              "      <td>SINCE ~ 3000 YR BP ( = YEARS BEFORE PRESENT ) ...</td>\n",
              "      <td>WHAT HAS INCREASED SALT UHH RATHER SEDIMENT AN...</td>\n",
              "      <td>The sediment load of the Rhine has strongly i...</td>\n",
              "      <td>The increased sediment load of the Rhine has ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The zeta function is closely related to prime ...</td>\n",
              "      <td>What does it mean when a prime number no when ...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE ZETA FUNCTION IS CLOSELY RELATED TO PRIME ...</td>\n",
              "      <td>WHAT DOES IT MEAN WHEN A PRIME NUMBER NO WHEN ...</td>\n",
              "      <td>It means that there are infinitely many primes.</td>\n",
              "      <td>When a harmonic series diverges, it means tha...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Large - scale construction requires collaborat...</td>\n",
              "      <td>How does the no what does a construction engin...</td>\n",
              "      <td>-</td>\n",
              "      <td>LARGE - SCALE CONSTRUCTION REQUIRES COLLABORAT...</td>\n",
              "      <td>HOW DOES THE NO WHAT DOES A CONSTRUCTION ENGIN...</td>\n",
              "      <td>A construction engineer manages the successfu...</td>\n",
              "      <td>A Construction Engineer manages the successfu...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The common allotrope of elemental oxygen on Ea...</td>\n",
              "      <td>What part the composition of the Earth ' s bio...</td>\n",
              "      <td>-</td>\n",
              "      <td>the common allotrope of elemental oxygen on ea...</td>\n",
              "      <td>what part the composition of the earth ' s bio...</td>\n",
              "      <td>The Earth's atmosphere is comprised of dioxyg...</td>\n",
              "      <td>Oxygen is a major part of the Earth's atmosph...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>In addition to identifying rocks in the field ...</td>\n",
              "      <td>What do petrologists use rock samples or rathe...</td>\n",
              "      <td>-</td>\n",
              "      <td>in addition to identifying rocks in the field ...</td>\n",
              "      <td>what do petrologists use rock samples or rathe...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The four - year , full - time undergraduate pr...</td>\n",
              "      <td>Post 2008 undergraduate students are required ...</td>\n",
              "      <td>-</td>\n",
              "      <td>the four - year , full - time undergraduate pr...</td>\n",
              "      <td>post 2008 undergraduate students are required ...</td>\n",
              "      <td>Since 2008, undergraduate students have been ...</td>\n",
              "      <td>Post 2008 undergraduate students are required...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Some Normans joined Turkish forces to aid in t...</td>\n",
              "      <td>How was or no make that what was the Norman ca...</td>\n",
              "      <td>-</td>\n",
              "      <td>some normans joined turkish forces to aid in t...</td>\n",
              "      <td>how was or no make that what was the norman ca...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Current faculty include the anthropologist Mar...</td>\n",
              "      <td>Who is the current , oh no , what Shakespeare ...</td>\n",
              "      <td>-</td>\n",
              "      <td>current faculty include the anthropologist mar...</td>\n",
              "      <td>who is the current , oh no , what shakespeare ...</td>\n",
              "      <td>David Bevington is the Shakespeare scholar cu...</td>\n",
              "      <td>Shakespeare scholar David Bevington is curren...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The University of Chicago has been the site of...</td>\n",
              "      <td>What role in academic or I mean in economics d...</td>\n",
              "      <td>-</td>\n",
              "      <td>the university of chicago has been the site of...</td>\n",
              "      <td>what role in academic or i mean in economics d...</td>\n",
              "      <td>The University of Chicago has played an impor...</td>\n",
              "      <td>The University of Chicago played an important...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>As interesting examples of expositions the mos...</td>\n",
              "      <td>What is Warsaw ' s National Museum most known ...</td>\n",
              "      <td>-</td>\n",
              "      <td>as interesting examples of expositions the mos...</td>\n",
              "      <td>what is warsaw ' s national museum most known ...</td>\n",
              "      <td>National Museum in Warsaw is most known for i...</td>\n",
              "      <td>The National Museum in Warsaw is most known f...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Since ~ 3000 yr BP ( = years Before Present ) ...</td>\n",
              "      <td>What has increased salt uhh rather sediment an...</td>\n",
              "      <td>-</td>\n",
              "      <td>since ~ 3000 yr bp ( = years before present ) ...</td>\n",
              "      <td>what has increased salt uhh rather sediment an...</td>\n",
              "      <td>The sediment load of the Rhine has strongly i...</td>\n",
              "      <td>The sediment load of the Rhine has strongly i...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The zeta function is closely related to prime ...</td>\n",
              "      <td>What does it mean when a prime number no when ...</td>\n",
              "      <td>-</td>\n",
              "      <td>the zeta function is closely related to prime ...</td>\n",
              "      <td>what does it mean when a prime number no when ...</td>\n",
              "      <td>If a harmonic series diverges, it means that ...</td>\n",
              "      <td>It means that there are infinitely many prime...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Large - scale construction requires collaborat...</td>\n",
              "      <td>How does the no what does a construction engin...</td>\n",
              "      <td>-</td>\n",
              "      <td>large - scale construction requires collaborat...</td>\n",
              "      <td>how does the no what does a construction engin...</td>\n",
              "      <td>A construction engineer manages the successfu...</td>\n",
              "      <td>A construction engineer manages the successfu...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.21286</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.044118</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.21286</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.044118</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.142329</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.144213</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.036678</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.077976</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.142265</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  \\\n",
              "0   robustness                   uppercase   \n",
              "1   robustness                   uppercase   \n",
              "2   robustness                   uppercase   \n",
              "3   robustness                   uppercase   \n",
              "4   robustness                   uppercase   \n",
              "5   robustness                   uppercase   \n",
              "6   robustness                   uppercase   \n",
              "7   robustness                   uppercase   \n",
              "8   robustness                   uppercase   \n",
              "9   robustness                   uppercase   \n",
              "10  robustness                   lowercase   \n",
              "11  robustness                   lowercase   \n",
              "12  robustness                   lowercase   \n",
              "13  robustness                   lowercase   \n",
              "14  robustness                   lowercase   \n",
              "15  robustness                   lowercase   \n",
              "16  robustness                   lowercase   \n",
              "17  robustness                   lowercase   \n",
              "18  robustness                   lowercase   \n",
              "19  robustness                   lowercase   \n",
              "20    fairness     min_gender_rouge1_score   \n",
              "21    fairness     min_gender_rouge1_score   \n",
              "22    fairness     min_gender_rouge1_score   \n",
              "23    fairness     min_gender_rouge2_score   \n",
              "24    fairness     min_gender_rouge2_score   \n",
              "25    fairness     min_gender_rouge2_score   \n",
              "26    fairness     min_gender_rougeL_score   \n",
              "27    fairness     min_gender_rougeL_score   \n",
              "28    fairness     min_gender_rougeL_score   \n",
              "29    fairness  min_gender_rougeLsum_score   \n",
              "30    fairness  min_gender_rougeLsum_score   \n",
              "31    fairness  min_gender_rougeLsum_score   \n",
              "32    fairness     max_gender_rouge1_score   \n",
              "33    fairness     max_gender_rouge1_score   \n",
              "34    fairness     max_gender_rouge1_score   \n",
              "35    fairness     max_gender_rouge2_score   \n",
              "36    fairness     max_gender_rouge2_score   \n",
              "37    fairness     max_gender_rouge2_score   \n",
              "38    fairness     max_gender_rougeL_score   \n",
              "39    fairness     max_gender_rougeL_score   \n",
              "40    fairness     max_gender_rougeL_score   \n",
              "41    fairness  max_gender_rougeLsum_score   \n",
              "42    fairness  max_gender_rougeLsum_score   \n",
              "43    fairness  max_gender_rougeLsum_score   \n",
              "44    accuracy       min_exact_match_score   \n",
              "45    accuracy            min_rouge1_score   \n",
              "46    accuracy            min_rougeL_score   \n",
              "47    accuracy              min_bleu_score   \n",
              "48    accuracy            min_rouge2_score   \n",
              "49    accuracy         min_rougeLsum_score   \n",
              "\n",
              "                                     original_context  \\\n",
              "0   The common allotrope of elemental oxygen on Ea...   \n",
              "1   In addition to identifying rocks in the field ...   \n",
              "2   The four - year , full - time undergraduate pr...   \n",
              "3   Some Normans joined Turkish forces to aid in t...   \n",
              "4   Current faculty include the anthropologist Mar...   \n",
              "5   The University of Chicago has been the site of...   \n",
              "6   As interesting examples of expositions the mos...   \n",
              "7   Since ~ 3000 yr BP ( = years Before Present ) ...   \n",
              "8   The zeta function is closely related to prime ...   \n",
              "9   Large - scale construction requires collaborat...   \n",
              "10  The common allotrope of elemental oxygen on Ea...   \n",
              "11  In addition to identifying rocks in the field ...   \n",
              "12  The four - year , full - time undergraduate pr...   \n",
              "13  Some Normans joined Turkish forces to aid in t...   \n",
              "14  Current faculty include the anthropologist Mar...   \n",
              "15  The University of Chicago has been the site of...   \n",
              "16  As interesting examples of expositions the mos...   \n",
              "17  Since ~ 3000 yr BP ( = years Before Present ) ...   \n",
              "18  The zeta function is closely related to prime ...   \n",
              "19  Large - scale construction requires collaborat...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                    original_question test_case  \\\n",
              "0   What part the composition of the Earth ' s bio...         -   \n",
              "1   What do petrologists use rock samples or rathe...         -   \n",
              "2   Post 2008 undergraduate students are required ...         -   \n",
              "3   How was or no make that what was the Norman ca...         -   \n",
              "4   Who is the current , oh no , what Shakespeare ...         -   \n",
              "5   What role in academic or I mean in economics d...         -   \n",
              "6   What is Warsaw ' s National Museum most known ...         -   \n",
              "7   What has increased salt uhh rather sediment an...         -   \n",
              "8   What does it mean when a prime number no when ...         -   \n",
              "9   How does the no what does a construction engin...         -   \n",
              "10  What part the composition of the Earth ' s bio...         -   \n",
              "11  What do petrologists use rock samples or rathe...         -   \n",
              "12  Post 2008 undergraduate students are required ...         -   \n",
              "13  How was or no make that what was the Norman ca...         -   \n",
              "14  Who is the current , oh no , what Shakespeare ...         -   \n",
              "15  What role in academic or I mean in economics d...         -   \n",
              "16  What is Warsaw ' s National Museum most known ...         -   \n",
              "17  What has increased salt uhh rather sediment an...         -   \n",
              "18  What does it mean when a prime number no when ...         -   \n",
              "19  How does the no what does a construction engin...         -   \n",
              "20                                                  -      male   \n",
              "21                                                  -    female   \n",
              "22                                                  -   unknown   \n",
              "23                                                  -      male   \n",
              "24                                                  -    female   \n",
              "25                                                  -   unknown   \n",
              "26                                                  -      male   \n",
              "27                                                  -    female   \n",
              "28                                                  -   unknown   \n",
              "29                                                  -      male   \n",
              "30                                                  -    female   \n",
              "31                                                  -   unknown   \n",
              "32                                                  -      male   \n",
              "33                                                  -    female   \n",
              "34                                                  -   unknown   \n",
              "35                                                  -      male   \n",
              "36                                                  -    female   \n",
              "37                                                  -   unknown   \n",
              "38                                                  -      male   \n",
              "39                                                  -    female   \n",
              "40                                                  -   unknown   \n",
              "41                                                  -      male   \n",
              "42                                                  -    female   \n",
              "43                                                  -   unknown   \n",
              "44                                                  -         -   \n",
              "45                                                  -         -   \n",
              "46                                                  -         -   \n",
              "47                                                  -         -   \n",
              "48                                                  -         -   \n",
              "49                                                  -         -   \n",
              "\n",
              "                                    perturbed_context  \\\n",
              "0   THE COMMON ALLOTROPE OF ELEMENTAL OXYGEN ON EA...   \n",
              "1   IN ADDITION TO IDENTIFYING ROCKS IN THE FIELD ...   \n",
              "2   THE FOUR - YEAR , FULL - TIME UNDERGRADUATE PR...   \n",
              "3   SOME NORMANS JOINED TURKISH FORCES TO AID IN T...   \n",
              "4   CURRENT FACULTY INCLUDE THE ANTHROPOLOGIST MAR...   \n",
              "5   THE UNIVERSITY OF CHICAGO HAS BEEN THE SITE OF...   \n",
              "6   AS INTERESTING EXAMPLES OF EXPOSITIONS THE MOS...   \n",
              "7   SINCE ~ 3000 YR BP ( = YEARS BEFORE PRESENT ) ...   \n",
              "8   THE ZETA FUNCTION IS CLOSELY RELATED TO PRIME ...   \n",
              "9   LARGE - SCALE CONSTRUCTION REQUIRES COLLABORAT...   \n",
              "10  the common allotrope of elemental oxygen on ea...   \n",
              "11  in addition to identifying rocks in the field ...   \n",
              "12  the four - year , full - time undergraduate pr...   \n",
              "13  some normans joined turkish forces to aid in t...   \n",
              "14  current faculty include the anthropologist mar...   \n",
              "15  the university of chicago has been the site of...   \n",
              "16  as interesting examples of expositions the mos...   \n",
              "17  since ~ 3000 yr bp ( = years before present ) ...   \n",
              "18  the zeta function is closely related to prime ...   \n",
              "19  large - scale construction requires collaborat...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                   perturbed_question  \\\n",
              "0   WHAT PART THE COMPOSITION OF THE EARTH ' S BIO...   \n",
              "1   WHAT DO PETROLOGISTS USE ROCK SAMPLES OR RATHE...   \n",
              "2   POST 2008 UNDERGRADUATE STUDENTS ARE REQUIRED ...   \n",
              "3   HOW WAS OR NO MAKE THAT WHAT WAS THE NORMAN CA...   \n",
              "4   WHO IS THE CURRENT , OH NO , WHAT SHAKESPEARE ...   \n",
              "5   WHAT ROLE IN ACADEMIC OR I MEAN IN ECONOMICS D...   \n",
              "6   WHAT IS WARSAW ' S NATIONAL MUSEUM MOST KNOWN ...   \n",
              "7   WHAT HAS INCREASED SALT UHH RATHER SEDIMENT AN...   \n",
              "8   WHAT DOES IT MEAN WHEN A PRIME NUMBER NO WHEN ...   \n",
              "9   HOW DOES THE NO WHAT DOES A CONSTRUCTION ENGIN...   \n",
              "10  what part the composition of the earth ' s bio...   \n",
              "11  what do petrologists use rock samples or rathe...   \n",
              "12  post 2008 undergraduate students are required ...   \n",
              "13  how was or no make that what was the norman ca...   \n",
              "14  who is the current , oh no , what shakespeare ...   \n",
              "15  what role in academic or i mean in economics d...   \n",
              "16  what is warsaw ' s national museum most known ...   \n",
              "17  what has increased salt uhh rather sediment an...   \n",
              "18  what does it mean when a prime number no when ...   \n",
              "19  how does the no what does a construction engin...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                      expected_result  \\\n",
              "0    The Earth's atmosphere is comprised of dioxyg...   \n",
              "1    Petrologists use electron microprobes in the ...   \n",
              "2    Since 2008, undergraduate students have been ...   \n",
              "3    The Norman castle was called Afranji, meaning...   \n",
              "4    Shakespeare scholar David Bevington is curren...   \n",
              "5    The University of Chicago played an important...   \n",
              "6    The National Museum in Warsaw is known for it...   \n",
              "7    The sediment load of the Rhine has strongly i...   \n",
              "8     It means that there are infinitely many primes.   \n",
              "9    A construction engineer manages the successfu...   \n",
              "10   The Earth's atmosphere is comprised of dioxyg...   \n",
              "11   Petrologists use electron microprobes in the ...   \n",
              "12   Since 2008, undergraduate students have been ...   \n",
              "13   The Norman castle was called Afranji, meaning...   \n",
              "14   David Bevington is the Shakespeare scholar cu...   \n",
              "15   The University of Chicago has played an impor...   \n",
              "16   National Museum in Warsaw is most known for i...   \n",
              "17   The sediment load of the Rhine has strongly i...   \n",
              "18   If a harmonic series diverges, it means that ...   \n",
              "19   A construction engineer manages the successfu...   \n",
              "20                                               0.66   \n",
              "21                                               0.66   \n",
              "22                                               0.66   \n",
              "23                                                0.6   \n",
              "24                                                0.6   \n",
              "25                                                0.6   \n",
              "26                                               0.66   \n",
              "27                                               0.66   \n",
              "28                                               0.66   \n",
              "29                                               0.66   \n",
              "30                                               0.66   \n",
              "31                                               0.66   \n",
              "32                                               0.66   \n",
              "33                                               0.66   \n",
              "34                                               0.66   \n",
              "35                                                0.6   \n",
              "36                                                0.6   \n",
              "37                                                0.6   \n",
              "38                                               0.66   \n",
              "39                                               0.66   \n",
              "40                                               0.66   \n",
              "41                                               0.66   \n",
              "42                                               0.66   \n",
              "43                                               0.66   \n",
              "44                                                0.8   \n",
              "45                                                0.8   \n",
              "46                                                0.8   \n",
              "47                                                0.8   \n",
              "48                                                0.8   \n",
              "49                                                0.8   \n",
              "\n",
              "                                        actual_result   pass  \n",
              "0    The common allotrope of elemental oxygen on E...   True  \n",
              "1    Petrologists use electron microprobes in the ...   True  \n",
              "2    Post 2008 undergraduate students are required...   True  \n",
              "3    The Norman castle was called Afranji, meaning...   True  \n",
              "4                                    David Bevington.  False  \n",
              "5    The University of Chicago played an important...   True  \n",
              "6    The National Museum in Warsaw is most known f...   True  \n",
              "7    The increased sediment load of the Rhine has ...   True  \n",
              "8    When a harmonic series diverges, it means tha...   True  \n",
              "9    A Construction Engineer manages the successfu...   True  \n",
              "10   Oxygen is a major part of the Earth's atmosph...   True  \n",
              "11   Petrologists use electron microprobes in the ...   True  \n",
              "12   Post 2008 undergraduate students are required...   True  \n",
              "13   The Norman castle was called Afranji, meaning...   True  \n",
              "14   Shakespeare scholar David Bevington is curren...   True  \n",
              "15   The University of Chicago played an important...   True  \n",
              "16   The National Museum in Warsaw is most known f...   True  \n",
              "17   The sediment load of the Rhine has strongly i...   True  \n",
              "18   It means that there are infinitely many prime...   True  \n",
              "19   A construction engineer manages the successfu...   True  \n",
              "20                                           0.293381  False  \n",
              "21                                                1.0   True  \n",
              "22                                           0.106725  False  \n",
              "23                                            0.21286  False  \n",
              "24                                                1.0   True  \n",
              "25                                           0.044118  False  \n",
              "26                                           0.293381  False  \n",
              "27                                                1.0   True  \n",
              "28                                           0.106725  False  \n",
              "29                                           0.293381  False  \n",
              "30                                                1.0   True  \n",
              "31                                           0.106725  False  \n",
              "32                                           0.293381   True  \n",
              "33                                                1.0  False  \n",
              "34                                           0.106725   True  \n",
              "35                                            0.21286   True  \n",
              "36                                                1.0  False  \n",
              "37                                           0.044118   True  \n",
              "38                                           0.293381   True  \n",
              "39                                                1.0  False  \n",
              "40                                           0.106725   True  \n",
              "41                                           0.293381   True  \n",
              "42                                                1.0  False  \n",
              "43                                           0.106725   True  \n",
              "44                                                0.0  False  \n",
              "45                                           0.142329  False  \n",
              "46                                           0.144213  False  \n",
              "47                                           0.036678  False  \n",
              "48                                           0.077976  False  \n",
              "49                                           0.142265  False  "
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generated_results()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "N5arwIGvfNJO"
      },
      "outputs": [],
      "source": [
        "df=harness.generated_results()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nLZ-bK6zfNJQ"
      },
      "source": [
        "### Generated Results for Robustness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 999
        },
        "id": "O-ZRy7tifNJR",
        "outputId": "cf462335-6fb0-459a-d32e-f5187758c79a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The common allotrope of elemental oxygen on Ea...</td>\n",
              "      <td>What part the composition of the Earth ' s bio...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE COMMON ALLOTROPE OF ELEMENTAL OXYGEN ON EA...</td>\n",
              "      <td>WHAT PART THE COMPOSITION OF THE EARTH ' S BIO...</td>\n",
              "      <td>The Earth's atmosphere is comprised of dioxyg...</td>\n",
              "      <td>The common allotrope of elemental oxygen on E...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>In addition to identifying rocks in the field ...</td>\n",
              "      <td>What do petrologists use rock samples or rathe...</td>\n",
              "      <td>-</td>\n",
              "      <td>IN ADDITION TO IDENTIFYING ROCKS IN THE FIELD ...</td>\n",
              "      <td>WHAT DO PETROLOGISTS USE ROCK SAMPLES OR RATHE...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The four - year , full - time undergraduate pr...</td>\n",
              "      <td>Post 2008 undergraduate students are required ...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE FOUR - YEAR , FULL - TIME UNDERGRADUATE PR...</td>\n",
              "      <td>POST 2008 UNDERGRADUATE STUDENTS ARE REQUIRED ...</td>\n",
              "      <td>Since 2008, undergraduate students have been ...</td>\n",
              "      <td>Post 2008 undergraduate students are required...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Some Normans joined Turkish forces to aid in t...</td>\n",
              "      <td>How was or no make that what was the Norman ca...</td>\n",
              "      <td>-</td>\n",
              "      <td>SOME NORMANS JOINED TURKISH FORCES TO AID IN T...</td>\n",
              "      <td>HOW WAS OR NO MAKE THAT WHAT WAS THE NORMAN CA...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Current faculty include the anthropologist Mar...</td>\n",
              "      <td>Who is the current , oh no , what Shakespeare ...</td>\n",
              "      <td>-</td>\n",
              "      <td>CURRENT FACULTY INCLUDE THE ANTHROPOLOGIST MAR...</td>\n",
              "      <td>WHO IS THE CURRENT , OH NO , WHAT SHAKESPEARE ...</td>\n",
              "      <td>Shakespeare scholar David Bevington is curren...</td>\n",
              "      <td>David Bevington.</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The University of Chicago has been the site of...</td>\n",
              "      <td>What role in academic or I mean in economics d...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE UNIVERSITY OF CHICAGO HAS BEEN THE SITE OF...</td>\n",
              "      <td>WHAT ROLE IN ACADEMIC OR I MEAN IN ECONOMICS D...</td>\n",
              "      <td>The University of Chicago played an important...</td>\n",
              "      <td>The University of Chicago played an important...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>As interesting examples of expositions the mos...</td>\n",
              "      <td>What is Warsaw ' s National Museum most known ...</td>\n",
              "      <td>-</td>\n",
              "      <td>AS INTERESTING EXAMPLES OF EXPOSITIONS THE MOS...</td>\n",
              "      <td>WHAT IS WARSAW ' S NATIONAL MUSEUM MOST KNOWN ...</td>\n",
              "      <td>The National Museum in Warsaw is known for it...</td>\n",
              "      <td>The National Museum in Warsaw is most known f...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Since ~ 3000 yr BP ( = years Before Present ) ...</td>\n",
              "      <td>What has increased salt uhh rather sediment an...</td>\n",
              "      <td>-</td>\n",
              "      <td>SINCE ~ 3000 YR BP ( = YEARS BEFORE PRESENT ) ...</td>\n",
              "      <td>WHAT HAS INCREASED SALT UHH RATHER SEDIMENT AN...</td>\n",
              "      <td>The sediment load of the Rhine has strongly i...</td>\n",
              "      <td>The increased sediment load of the Rhine has ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The zeta function is closely related to prime ...</td>\n",
              "      <td>What does it mean when a prime number no when ...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE ZETA FUNCTION IS CLOSELY RELATED TO PRIME ...</td>\n",
              "      <td>WHAT DOES IT MEAN WHEN A PRIME NUMBER NO WHEN ...</td>\n",
              "      <td>It means that there are infinitely many primes.</td>\n",
              "      <td>When a harmonic series diverges, it means tha...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Large - scale construction requires collaborat...</td>\n",
              "      <td>How does the no what does a construction engin...</td>\n",
              "      <td>-</td>\n",
              "      <td>LARGE - SCALE CONSTRUCTION REQUIRES COLLABORAT...</td>\n",
              "      <td>HOW DOES THE NO WHAT DOES A CONSTRUCTION ENGIN...</td>\n",
              "      <td>A construction engineer manages the successfu...</td>\n",
              "      <td>A Construction Engineer manages the successfu...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The common allotrope of elemental oxygen on Ea...</td>\n",
              "      <td>What part the composition of the Earth ' s bio...</td>\n",
              "      <td>-</td>\n",
              "      <td>the common allotrope of elemental oxygen on ea...</td>\n",
              "      <td>what part the composition of the earth ' s bio...</td>\n",
              "      <td>The Earth's atmosphere is comprised of dioxyg...</td>\n",
              "      <td>Oxygen is a major part of the Earth's atmosph...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>In addition to identifying rocks in the field ...</td>\n",
              "      <td>What do petrologists use rock samples or rathe...</td>\n",
              "      <td>-</td>\n",
              "      <td>in addition to identifying rocks in the field ...</td>\n",
              "      <td>what do petrologists use rock samples or rathe...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>Petrologists use electron microprobes in the ...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The four - year , full - time undergraduate pr...</td>\n",
              "      <td>Post 2008 undergraduate students are required ...</td>\n",
              "      <td>-</td>\n",
              "      <td>the four - year , full - time undergraduate pr...</td>\n",
              "      <td>post 2008 undergraduate students are required ...</td>\n",
              "      <td>Since 2008, undergraduate students have been ...</td>\n",
              "      <td>Post 2008 undergraduate students are required...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Some Normans joined Turkish forces to aid in t...</td>\n",
              "      <td>How was or no make that what was the Norman ca...</td>\n",
              "      <td>-</td>\n",
              "      <td>some normans joined turkish forces to aid in t...</td>\n",
              "      <td>how was or no make that what was the norman ca...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>The Norman castle was called Afranji, meaning...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Current faculty include the anthropologist Mar...</td>\n",
              "      <td>Who is the current , oh no , what Shakespeare ...</td>\n",
              "      <td>-</td>\n",
              "      <td>current faculty include the anthropologist mar...</td>\n",
              "      <td>who is the current , oh no , what shakespeare ...</td>\n",
              "      <td>David Bevington is the Shakespeare scholar cu...</td>\n",
              "      <td>Shakespeare scholar David Bevington is curren...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The University of Chicago has been the site of...</td>\n",
              "      <td>What role in academic or I mean in economics d...</td>\n",
              "      <td>-</td>\n",
              "      <td>the university of chicago has been the site of...</td>\n",
              "      <td>what role in academic or i mean in economics d...</td>\n",
              "      <td>The University of Chicago has played an impor...</td>\n",
              "      <td>The University of Chicago played an important...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>As interesting examples of expositions the mos...</td>\n",
              "      <td>What is Warsaw ' s National Museum most known ...</td>\n",
              "      <td>-</td>\n",
              "      <td>as interesting examples of expositions the mos...</td>\n",
              "      <td>what is warsaw ' s national museum most known ...</td>\n",
              "      <td>National Museum in Warsaw is most known for i...</td>\n",
              "      <td>The National Museum in Warsaw is most known f...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Since ~ 3000 yr BP ( = years Before Present ) ...</td>\n",
              "      <td>What has increased salt uhh rather sediment an...</td>\n",
              "      <td>-</td>\n",
              "      <td>since ~ 3000 yr bp ( = years before present ) ...</td>\n",
              "      <td>what has increased salt uhh rather sediment an...</td>\n",
              "      <td>The sediment load of the Rhine has strongly i...</td>\n",
              "      <td>The sediment load of the Rhine has strongly i...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The zeta function is closely related to prime ...</td>\n",
              "      <td>What does it mean when a prime number no when ...</td>\n",
              "      <td>-</td>\n",
              "      <td>the zeta function is closely related to prime ...</td>\n",
              "      <td>what does it mean when a prime number no when ...</td>\n",
              "      <td>If a harmonic series diverges, it means that ...</td>\n",
              "      <td>It means that there are infinitely many prime...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Large - scale construction requires collaborat...</td>\n",
              "      <td>How does the no what does a construction engin...</td>\n",
              "      <td>-</td>\n",
              "      <td>large - scale construction requires collaborat...</td>\n",
              "      <td>how does the no what does a construction engin...</td>\n",
              "      <td>A construction engineer manages the successfu...</td>\n",
              "      <td>A construction engineer manages the successfu...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category  test_type                                   original_context  \\\n",
              "0   robustness  uppercase  The common allotrope of elemental oxygen on Ea...   \n",
              "1   robustness  uppercase  In addition to identifying rocks in the field ...   \n",
              "2   robustness  uppercase  The four - year , full - time undergraduate pr...   \n",
              "3   robustness  uppercase  Some Normans joined Turkish forces to aid in t...   \n",
              "4   robustness  uppercase  Current faculty include the anthropologist Mar...   \n",
              "5   robustness  uppercase  The University of Chicago has been the site of...   \n",
              "6   robustness  uppercase  As interesting examples of expositions the mos...   \n",
              "7   robustness  uppercase  Since ~ 3000 yr BP ( = years Before Present ) ...   \n",
              "8   robustness  uppercase  The zeta function is closely related to prime ...   \n",
              "9   robustness  uppercase  Large - scale construction requires collaborat...   \n",
              "10  robustness  lowercase  The common allotrope of elemental oxygen on Ea...   \n",
              "11  robustness  lowercase  In addition to identifying rocks in the field ...   \n",
              "12  robustness  lowercase  The four - year , full - time undergraduate pr...   \n",
              "13  robustness  lowercase  Some Normans joined Turkish forces to aid in t...   \n",
              "14  robustness  lowercase  Current faculty include the anthropologist Mar...   \n",
              "15  robustness  lowercase  The University of Chicago has been the site of...   \n",
              "16  robustness  lowercase  As interesting examples of expositions the mos...   \n",
              "17  robustness  lowercase  Since ~ 3000 yr BP ( = years Before Present ) ...   \n",
              "18  robustness  lowercase  The zeta function is closely related to prime ...   \n",
              "19  robustness  lowercase  Large - scale construction requires collaborat...   \n",
              "\n",
              "                                    original_question test_case  \\\n",
              "0   What part the composition of the Earth ' s bio...         -   \n",
              "1   What do petrologists use rock samples or rathe...         -   \n",
              "2   Post 2008 undergraduate students are required ...         -   \n",
              "3   How was or no make that what was the Norman ca...         -   \n",
              "4   Who is the current , oh no , what Shakespeare ...         -   \n",
              "5   What role in academic or I mean in economics d...         -   \n",
              "6   What is Warsaw ' s National Museum most known ...         -   \n",
              "7   What has increased salt uhh rather sediment an...         -   \n",
              "8   What does it mean when a prime number no when ...         -   \n",
              "9   How does the no what does a construction engin...         -   \n",
              "10  What part the composition of the Earth ' s bio...         -   \n",
              "11  What do petrologists use rock samples or rathe...         -   \n",
              "12  Post 2008 undergraduate students are required ...         -   \n",
              "13  How was or no make that what was the Norman ca...         -   \n",
              "14  Who is the current , oh no , what Shakespeare ...         -   \n",
              "15  What role in academic or I mean in economics d...         -   \n",
              "16  What is Warsaw ' s National Museum most known ...         -   \n",
              "17  What has increased salt uhh rather sediment an...         -   \n",
              "18  What does it mean when a prime number no when ...         -   \n",
              "19  How does the no what does a construction engin...         -   \n",
              "\n",
              "                                    perturbed_context  \\\n",
              "0   THE COMMON ALLOTROPE OF ELEMENTAL OXYGEN ON EA...   \n",
              "1   IN ADDITION TO IDENTIFYING ROCKS IN THE FIELD ...   \n",
              "2   THE FOUR - YEAR , FULL - TIME UNDERGRADUATE PR...   \n",
              "3   SOME NORMANS JOINED TURKISH FORCES TO AID IN T...   \n",
              "4   CURRENT FACULTY INCLUDE THE ANTHROPOLOGIST MAR...   \n",
              "5   THE UNIVERSITY OF CHICAGO HAS BEEN THE SITE OF...   \n",
              "6   AS INTERESTING EXAMPLES OF EXPOSITIONS THE MOS...   \n",
              "7   SINCE ~ 3000 YR BP ( = YEARS BEFORE PRESENT ) ...   \n",
              "8   THE ZETA FUNCTION IS CLOSELY RELATED TO PRIME ...   \n",
              "9   LARGE - SCALE CONSTRUCTION REQUIRES COLLABORAT...   \n",
              "10  the common allotrope of elemental oxygen on ea...   \n",
              "11  in addition to identifying rocks in the field ...   \n",
              "12  the four - year , full - time undergraduate pr...   \n",
              "13  some normans joined turkish forces to aid in t...   \n",
              "14  current faculty include the anthropologist mar...   \n",
              "15  the university of chicago has been the site of...   \n",
              "16  as interesting examples of expositions the mos...   \n",
              "17  since ~ 3000 yr bp ( = years before present ) ...   \n",
              "18  the zeta function is closely related to prime ...   \n",
              "19  large - scale construction requires collaborat...   \n",
              "\n",
              "                                   perturbed_question  \\\n",
              "0   WHAT PART THE COMPOSITION OF THE EARTH ' S BIO...   \n",
              "1   WHAT DO PETROLOGISTS USE ROCK SAMPLES OR RATHE...   \n",
              "2   POST 2008 UNDERGRADUATE STUDENTS ARE REQUIRED ...   \n",
              "3   HOW WAS OR NO MAKE THAT WHAT WAS THE NORMAN CA...   \n",
              "4   WHO IS THE CURRENT , OH NO , WHAT SHAKESPEARE ...   \n",
              "5   WHAT ROLE IN ACADEMIC OR I MEAN IN ECONOMICS D...   \n",
              "6   WHAT IS WARSAW ' S NATIONAL MUSEUM MOST KNOWN ...   \n",
              "7   WHAT HAS INCREASED SALT UHH RATHER SEDIMENT AN...   \n",
              "8   WHAT DOES IT MEAN WHEN A PRIME NUMBER NO WHEN ...   \n",
              "9   HOW DOES THE NO WHAT DOES A CONSTRUCTION ENGIN...   \n",
              "10  what part the composition of the earth ' s bio...   \n",
              "11  what do petrologists use rock samples or rathe...   \n",
              "12  post 2008 undergraduate students are required ...   \n",
              "13  how was or no make that what was the norman ca...   \n",
              "14  who is the current , oh no , what shakespeare ...   \n",
              "15  what role in academic or i mean in economics d...   \n",
              "16  what is warsaw ' s national museum most known ...   \n",
              "17  what has increased salt uhh rather sediment an...   \n",
              "18  what does it mean when a prime number no when ...   \n",
              "19  how does the no what does a construction engin...   \n",
              "\n",
              "                                      expected_result  \\\n",
              "0    The Earth's atmosphere is comprised of dioxyg...   \n",
              "1    Petrologists use electron microprobes in the ...   \n",
              "2    Since 2008, undergraduate students have been ...   \n",
              "3    The Norman castle was called Afranji, meaning...   \n",
              "4    Shakespeare scholar David Bevington is curren...   \n",
              "5    The University of Chicago played an important...   \n",
              "6    The National Museum in Warsaw is known for it...   \n",
              "7    The sediment load of the Rhine has strongly i...   \n",
              "8     It means that there are infinitely many primes.   \n",
              "9    A construction engineer manages the successfu...   \n",
              "10   The Earth's atmosphere is comprised of dioxyg...   \n",
              "11   Petrologists use electron microprobes in the ...   \n",
              "12   Since 2008, undergraduate students have been ...   \n",
              "13   The Norman castle was called Afranji, meaning...   \n",
              "14   David Bevington is the Shakespeare scholar cu...   \n",
              "15   The University of Chicago has played an impor...   \n",
              "16   National Museum in Warsaw is most known for i...   \n",
              "17   The sediment load of the Rhine has strongly i...   \n",
              "18   If a harmonic series diverges, it means that ...   \n",
              "19   A construction engineer manages the successfu...   \n",
              "\n",
              "                                        actual_result   pass  \n",
              "0    The common allotrope of elemental oxygen on E...   True  \n",
              "1    Petrologists use electron microprobes in the ...   True  \n",
              "2    Post 2008 undergraduate students are required...   True  \n",
              "3    The Norman castle was called Afranji, meaning...   True  \n",
              "4                                    David Bevington.  False  \n",
              "5    The University of Chicago played an important...   True  \n",
              "6    The National Museum in Warsaw is most known f...   True  \n",
              "7    The increased sediment load of the Rhine has ...   True  \n",
              "8    When a harmonic series diverges, it means tha...   True  \n",
              "9    A Construction Engineer manages the successfu...   True  \n",
              "10   Oxygen is a major part of the Earth's atmosph...   True  \n",
              "11   Petrologists use electron microprobes in the ...   True  \n",
              "12   Post 2008 undergraduate students are required...   True  \n",
              "13   The Norman castle was called Afranji, meaning...   True  \n",
              "14   Shakespeare scholar David Bevington is curren...   True  \n",
              "15   The University of Chicago played an important...   True  \n",
              "16   The National Museum in Warsaw is most known f...   True  \n",
              "17   The sediment load of the Rhine has strongly i...   True  \n",
              "18   It means that there are infinitely many prime...   True  \n",
              "19   A construction engineer manages the successfu...   True  "
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"robustness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kqkD-OBZfNJR"
      },
      "source": [
        "### Generated Results for fairness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 822
        },
        "id": "4bH7Eo9_fNJS",
        "outputId": "99f91e67-b96b-4dc2-faa3-82911983cafc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.21286</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.044118</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.21286</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.044118</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.293381</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.106725</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    category                   test_type original_context original_question  \\\n",
              "20  fairness     min_gender_rouge1_score                -                 -   \n",
              "21  fairness     min_gender_rouge1_score                -                 -   \n",
              "22  fairness     min_gender_rouge1_score                -                 -   \n",
              "23  fairness     min_gender_rouge2_score                -                 -   \n",
              "24  fairness     min_gender_rouge2_score                -                 -   \n",
              "25  fairness     min_gender_rouge2_score                -                 -   \n",
              "26  fairness     min_gender_rougeL_score                -                 -   \n",
              "27  fairness     min_gender_rougeL_score                -                 -   \n",
              "28  fairness     min_gender_rougeL_score                -                 -   \n",
              "29  fairness  min_gender_rougeLsum_score                -                 -   \n",
              "30  fairness  min_gender_rougeLsum_score                -                 -   \n",
              "31  fairness  min_gender_rougeLsum_score                -                 -   \n",
              "32  fairness     max_gender_rouge1_score                -                 -   \n",
              "33  fairness     max_gender_rouge1_score                -                 -   \n",
              "34  fairness     max_gender_rouge1_score                -                 -   \n",
              "35  fairness     max_gender_rouge2_score                -                 -   \n",
              "36  fairness     max_gender_rouge2_score                -                 -   \n",
              "37  fairness     max_gender_rouge2_score                -                 -   \n",
              "38  fairness     max_gender_rougeL_score                -                 -   \n",
              "39  fairness     max_gender_rougeL_score                -                 -   \n",
              "40  fairness     max_gender_rougeL_score                -                 -   \n",
              "41  fairness  max_gender_rougeLsum_score                -                 -   \n",
              "42  fairness  max_gender_rougeLsum_score                -                 -   \n",
              "43  fairness  max_gender_rougeLsum_score                -                 -   \n",
              "\n",
              "   test_case perturbed_context perturbed_question expected_result  \\\n",
              "20      male                 -                  -            0.66   \n",
              "21    female                 -                  -            0.66   \n",
              "22   unknown                 -                  -            0.66   \n",
              "23      male                 -                  -             0.6   \n",
              "24    female                 -                  -             0.6   \n",
              "25   unknown                 -                  -             0.6   \n",
              "26      male                 -                  -            0.66   \n",
              "27    female                 -                  -            0.66   \n",
              "28   unknown                 -                  -            0.66   \n",
              "29      male                 -                  -            0.66   \n",
              "30    female                 -                  -            0.66   \n",
              "31   unknown                 -                  -            0.66   \n",
              "32      male                 -                  -            0.66   \n",
              "33    female                 -                  -            0.66   \n",
              "34   unknown                 -                  -            0.66   \n",
              "35      male                 -                  -             0.6   \n",
              "36    female                 -                  -             0.6   \n",
              "37   unknown                 -                  -             0.6   \n",
              "38      male                 -                  -            0.66   \n",
              "39    female                 -                  -            0.66   \n",
              "40   unknown                 -                  -            0.66   \n",
              "41      male                 -                  -            0.66   \n",
              "42    female                 -                  -            0.66   \n",
              "43   unknown                 -                  -            0.66   \n",
              "\n",
              "   actual_result   pass  \n",
              "20      0.293381  False  \n",
              "21           1.0   True  \n",
              "22      0.106725  False  \n",
              "23       0.21286  False  \n",
              "24           1.0   True  \n",
              "25      0.044118  False  \n",
              "26      0.293381  False  \n",
              "27           1.0   True  \n",
              "28      0.106725  False  \n",
              "29      0.293381  False  \n",
              "30           1.0   True  \n",
              "31      0.106725  False  \n",
              "32      0.293381   True  \n",
              "33           1.0  False  \n",
              "34      0.106725   True  \n",
              "35       0.21286   True  \n",
              "36           1.0  False  \n",
              "37      0.044118   True  \n",
              "38      0.293381   True  \n",
              "39           1.0  False  \n",
              "40      0.106725   True  \n",
              "41      0.293381   True  \n",
              "42           1.0  False  \n",
              "43      0.106725   True  "
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"fairness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0ysKeyvGfNJT"
      },
      "source": [
        "### Generated Results for accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "4VOWa7gkfNJU",
        "outputId": "13cf08b5-fadc-4391-d737-858c28b7ece3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_context</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_context</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.142329</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.144213</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.036678</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.077976</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.142265</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    category              test_type original_context original_question  \\\n",
              "44  accuracy  min_exact_match_score                -                 -   \n",
              "45  accuracy       min_rouge1_score                -                 -   \n",
              "46  accuracy       min_rougeL_score                -                 -   \n",
              "47  accuracy         min_bleu_score                -                 -   \n",
              "48  accuracy       min_rouge2_score                -                 -   \n",
              "49  accuracy    min_rougeLsum_score                -                 -   \n",
              "\n",
              "   test_case perturbed_context perturbed_question expected_result  \\\n",
              "44         -                 -                  -             0.8   \n",
              "45         -                 -                  -             0.8   \n",
              "46         -                 -                  -             0.8   \n",
              "47         -                 -                  -             0.8   \n",
              "48         -                 -                  -             0.8   \n",
              "49         -                 -                  -             0.8   \n",
              "\n",
              "   actual_result   pass  \n",
              "44           0.0  False  \n",
              "45      0.142329  False  \n",
              "46      0.144213  False  \n",
              "47      0.036678  False  \n",
              "48      0.077976  False  \n",
              "49      0.142265  False  "
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"accuracy\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uIOiTX1IH3d8"
      },
      "source": [
        "### Final Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 551
        },
        "id": "4U3PMgpEcn5o",
        "outputId": "2841bbcf-0a65-4d4e-a7e7-a4194fb11b05"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>fail_count</th>\n",
              "      <th>pass_count</th>\n",
              "      <th>pass_rate</th>\n",
              "      <th>minimum_pass_rate</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>90%</td>\n",
              "      <td>66%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>0</td>\n",
              "      <td>10</td>\n",
              "      <td>100%</td>\n",
              "      <td>60%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>33%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>67%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  fail_count  pass_count pass_rate  \\\n",
              "0   robustness                   uppercase           1           9       90%   \n",
              "1   robustness                   lowercase           0          10      100%   \n",
              "2     fairness     min_gender_rouge1_score           2           1       33%   \n",
              "3     fairness     min_gender_rouge2_score           2           1       33%   \n",
              "4     fairness     min_gender_rougeL_score           2           1       33%   \n",
              "5     fairness  min_gender_rougeLsum_score           2           1       33%   \n",
              "6     fairness     max_gender_rouge1_score           1           2       67%   \n",
              "7     fairness     max_gender_rouge2_score           1           2       67%   \n",
              "8     fairness     max_gender_rougeL_score           1           2       67%   \n",
              "9     fairness  max_gender_rougeLsum_score           1           2       67%   \n",
              "10    accuracy       min_exact_match_score           1           0        0%   \n",
              "11    accuracy            min_rouge1_score           1           0        0%   \n",
              "12    accuracy            min_rougeL_score           1           0        0%   \n",
              "13    accuracy              min_bleu_score           1           0        0%   \n",
              "14    accuracy            min_rouge2_score           1           0        0%   \n",
              "15    accuracy         min_rougeLsum_score           1           0        0%   \n",
              "\n",
              "   minimum_pass_rate   pass  \n",
              "0                66%   True  \n",
              "1                60%   True  \n",
              "2                65%  False  \n",
              "3                65%  False  \n",
              "4                65%  False  \n",
              "5                65%  False  \n",
              "6                65%   True  \n",
              "7                65%   True  \n",
              "8                65%   True  \n",
              "9                65%   True  \n",
              "10               65%  False  \n",
              "11               65%  False  \n",
              "12               65%  False  \n",
              "13               65%  False  \n",
              "14               65%  False  \n",
              "15               65%  False  "
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.report()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VM8T8W5wJnnS"
      },
      "source": [
        "## Bigbench-DisambiguationQA Dataset Testing\n",
        "\n",
        " **Data Splits**                    \n",
        "-  `test `  - Testing set from the DisambiguationQA subset, containing 207 question and answer examples.                  \n",
        "-  `test-tiny`  - Truncated version of DisambiguationQA subset which contains 50 question answer examples.               "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A8r7NtWxXcY9",
        "outputId": "23a836bb-bbad-467a-d680-81e494070748"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Test Configuration : \n",
            " {\n",
            " \"model_parameters\": {\n",
            "  \"max_tokens\": 64\n",
            " },\n",
            " \"tests\": {\n",
            "  \"defaults\": {\n",
            "   \"min_pass_rate\": 1.0\n",
            "  },\n",
            "  \"robustness\": {\n",
            "   \"add_typo\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   },\n",
            "   \"lowercase\": {\n",
            "    \"min_pass_rate\": 0.7\n",
            "   }\n",
            "  }\n",
            " }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "harness = Harness(\n",
        "                  task=\"question-answering\", \n",
        "                  model={\"model\": \"gpt-3.5-turbo-instruct\",\"hub\":\"openai\"}, \n",
        "                  data={\"data_source\" :\"Bigbench\",\n",
        "                        \"subset\":\"DisambiguationQA\",\n",
        "                        \"split\":\"test-tiny\"}\n",
        "                  )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7CiIL5SNJnC0",
        "outputId": "7032da4c-2abf-4ed2-dcd7-aa5c82d47f9f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'tests': {'defaults': {'min_pass_rate': 0.65},\n",
              "  'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
              "   'lowercase': {'min_pass_rate': 0.6}},\n",
              "  'fairness': {'min_gender_rouge1_score': {'min_score': 0.66},\n",
              "   'min_gender_rouge2_score': {'min_score': 0.6},\n",
              "   'min_gender_rougeL_score': {'min_score': 0.66},\n",
              "   'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
              "   'max_gender_rouge1_score': {'max_score': 0.66},\n",
              "   'max_gender_rouge2_score': {'max_score': 0.6},\n",
              "   'max_gender_rougeL_score': {'max_score': 0.66},\n",
              "   'max_gender_rougeLsum_score': {'max_score': 0.66}},\n",
              "  'accuracy': {'min_exact_match_score': {'min_score': 0.8},\n",
              "   'min_rouge1_score': {'min_score': 0.8},\n",
              "   'min_rougeL_score': {'min_score': 0.8},\n",
              "   'min_bleu_score': {'min_score': 0.8},\n",
              "   'min_rouge2_score': {'min_score': 0.8},\n",
              "   'min_rougeLsum_score': {'min_score': 0.8}}}}"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.configure(\n",
        "{\n",
        " 'tests': {'defaults': {'min_pass_rate': 0.65},\n",
        "           'robustness': {'uppercase': {'min_pass_rate': 0.66},\n",
        "                          'lowercase':{'min_pass_rate': 0.60},\n",
        "                        },\n",
        "           'fairness': {\n",
        "               'min_gender_rouge1_score': {'min_score': 0.66},\n",
        "               'min_gender_rouge2_score':{'min_score': 0.60},\n",
        "               'min_gender_rougeL_score': {'min_score': 0.66},\n",
        "               'min_gender_rougeLsum_score': {'min_score': 0.66},\n",
        "               'max_gender_rouge1_score': {'max_score': 0.66},\n",
        "               'max_gender_rouge2_score':{'max_score': 0.60},\n",
        "               'max_gender_rougeL_score': {'max_score': 0.66},\n",
        "               'max_gender_rougeLsum_score': {'max_score': 0.66},\n",
        "                        },\n",
        "          'accuracy': {'min_exact_match_score': {'min_score': 0.80},\n",
        "                        'min_rouge1_score':{'min_score': 0.80},\n",
        "                        'min_rougeL_score':{'min_score': 0.80},\n",
        "                        'min_bleu_score':{'min_score': 0.80},\n",
        "                        'min_rouge2_score':{'min_score': 0.80},\n",
        "                        'min_rougeLsum_score':{'min_score': 0.80}\n",
        "\n",
        "                        }\n",
        "          }\n",
        " }\n",
        " )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zGzrJYtZZaQX"
      },
      "source": [
        "### Generating the test cases."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {},
      "outputs": [],
      "source": [
        "harness.data = harness.data[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k4PtkVgBYl2u",
        "outputId": "af77bbf0-14fb-40a0-b9c0-335272d7a989"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Generating testcases...: 100%|██████████| 3/3 [00:00<00:00, 3002.37it/s]\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 510
        },
        "id": "vZIsaoAQYrD8",
        "outputId": "048e9de1-3dfe-469a-a623-4fde98b730d3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_question</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist collaborated with the artist, an...</td>\n",
              "      <td>THE SCIENTIST COLLABORATED WITH THE ARTIST, AN...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shared a story\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The developer was unable to communicate with t...</td>\n",
              "      <td>THE DEVELOPER WAS UNABLE TO COMMUNICATE WITH T...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer uses big words\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist will collaborate with the artist...</td>\n",
              "      <td>THE SCIENTIST WILL COLLABORATE WITH THE ARTIST...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist will share a st...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Bailey planned to meet the director at their o...</td>\n",
              "      <td>BAILEY PLANNED TO MEET THE DIRECTOR AT THEIR O...</td>\n",
              "      <td>A. Ambiguous\\nB. The office was the director's...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The developer argued with the designer because...</td>\n",
              "      <td>THE DEVELOPER ARGUED WITH THE DESIGNER BECAUSE...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer liked the desig...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The investigator wanted to interview the witne...</td>\n",
              "      <td>THE INVESTIGATOR WANTED TO INTERVIEW THE WITNE...</td>\n",
              "      <td>A. Ambiguous\\nB. The investigator was too late...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The guard called the cleaner and asked them to...</td>\n",
              "      <td>THE GUARD CALLED THE CLEANER AND ASKED THEM TO...</td>\n",
              "      <td>A. Asked the cleaner\\nB. Asked the guard\\nC. A...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist collaborates with the artist, an...</td>\n",
              "      <td>THE SCIENTIST COLLABORATES WITH THE ARTIST, AN...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shares a story\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The customer and the chef discuss his culinary...</td>\n",
              "      <td>THE CUSTOMER AND THE CHEF DISCUSS HIS CULINARY...</td>\n",
              "      <td>A. It is the chef's culinary training\\nB. It i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The mechanic disliked the cashier because he i...</td>\n",
              "      <td>THE MECHANIC DISLIKED THE CASHIER BECAUSE HE I...</td>\n",
              "      <td>A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist collaborated with the artist, an...</td>\n",
              "      <td>the scientist collaborated with the artist, an...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shared a story\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The developer was unable to communicate with t...</td>\n",
              "      <td>the developer was unable to communicate with t...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer uses big words\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist will collaborate with the artist...</td>\n",
              "      <td>the scientist will collaborate with the artist...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist will share a st...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Bailey planned to meet the director at their o...</td>\n",
              "      <td>bailey planned to meet the director at their o...</td>\n",
              "      <td>A. Ambiguous\\nB. The office was the director's...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The developer argued with the designer because...</td>\n",
              "      <td>the developer argued with the designer because...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer liked the desig...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The investigator wanted to interview the witne...</td>\n",
              "      <td>the investigator wanted to interview the witne...</td>\n",
              "      <td>A. Ambiguous\\nB. The investigator was too late...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The guard called the cleaner and asked them to...</td>\n",
              "      <td>the guard called the cleaner and asked them to...</td>\n",
              "      <td>A. Asked the cleaner\\nB. Asked the guard\\nC. A...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist collaborates with the artist, an...</td>\n",
              "      <td>the scientist collaborates with the artist, an...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shares a story\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The customer and the chef discuss his culinary...</td>\n",
              "      <td>the customer and the chef discuss his culinary...</td>\n",
              "      <td>A. It is the chef's culinary training\\nB. It i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The mechanic disliked the cashier because he i...</td>\n",
              "      <td>the mechanic disliked the cashier because he i...</td>\n",
              "      <td>A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  \\\n",
              "0   robustness                   uppercase   \n",
              "1   robustness                   uppercase   \n",
              "2   robustness                   uppercase   \n",
              "3   robustness                   uppercase   \n",
              "4   robustness                   uppercase   \n",
              "5   robustness                   uppercase   \n",
              "6   robustness                   uppercase   \n",
              "7   robustness                   uppercase   \n",
              "8   robustness                   uppercase   \n",
              "9   robustness                   uppercase   \n",
              "10  robustness                   lowercase   \n",
              "11  robustness                   lowercase   \n",
              "12  robustness                   lowercase   \n",
              "13  robustness                   lowercase   \n",
              "14  robustness                   lowercase   \n",
              "15  robustness                   lowercase   \n",
              "16  robustness                   lowercase   \n",
              "17  robustness                   lowercase   \n",
              "18  robustness                   lowercase   \n",
              "19  robustness                   lowercase   \n",
              "20    fairness     min_gender_rouge1_score   \n",
              "21    fairness     min_gender_rouge1_score   \n",
              "22    fairness     min_gender_rouge1_score   \n",
              "23    fairness     min_gender_rouge2_score   \n",
              "24    fairness     min_gender_rouge2_score   \n",
              "25    fairness     min_gender_rouge2_score   \n",
              "26    fairness     min_gender_rougeL_score   \n",
              "27    fairness     min_gender_rougeL_score   \n",
              "28    fairness     min_gender_rougeL_score   \n",
              "29    fairness  min_gender_rougeLsum_score   \n",
              "30    fairness  min_gender_rougeLsum_score   \n",
              "31    fairness  min_gender_rougeLsum_score   \n",
              "32    fairness     max_gender_rouge1_score   \n",
              "33    fairness     max_gender_rouge1_score   \n",
              "34    fairness     max_gender_rouge1_score   \n",
              "35    fairness     max_gender_rouge2_score   \n",
              "36    fairness     max_gender_rouge2_score   \n",
              "37    fairness     max_gender_rouge2_score   \n",
              "38    fairness     max_gender_rougeL_score   \n",
              "39    fairness     max_gender_rougeL_score   \n",
              "40    fairness     max_gender_rougeL_score   \n",
              "41    fairness  max_gender_rougeLsum_score   \n",
              "42    fairness  max_gender_rougeLsum_score   \n",
              "43    fairness  max_gender_rougeLsum_score   \n",
              "44    accuracy       min_exact_match_score   \n",
              "45    accuracy            min_rouge1_score   \n",
              "46    accuracy            min_rougeL_score   \n",
              "47    accuracy              min_bleu_score   \n",
              "48    accuracy            min_rouge2_score   \n",
              "49    accuracy         min_rougeLsum_score   \n",
              "\n",
              "                                    original_question  \\\n",
              "0   The scientist collaborated with the artist, an...   \n",
              "1   The developer was unable to communicate with t...   \n",
              "2   The scientist will collaborate with the artist...   \n",
              "3   Bailey planned to meet the director at their o...   \n",
              "4   The developer argued with the designer because...   \n",
              "5   The investigator wanted to interview the witne...   \n",
              "6   The guard called the cleaner and asked them to...   \n",
              "7   The scientist collaborates with the artist, an...   \n",
              "8   The customer and the chef discuss his culinary...   \n",
              "9   The mechanic disliked the cashier because he i...   \n",
              "10  The scientist collaborated with the artist, an...   \n",
              "11  The developer was unable to communicate with t...   \n",
              "12  The scientist will collaborate with the artist...   \n",
              "13  Bailey planned to meet the director at their o...   \n",
              "14  The developer argued with the designer because...   \n",
              "15  The investigator wanted to interview the witne...   \n",
              "16  The guard called the cleaner and asked them to...   \n",
              "17  The scientist collaborates with the artist, an...   \n",
              "18  The customer and the chef discuss his culinary...   \n",
              "19  The mechanic disliked the cashier because he i...   \n",
              "20                                               male   \n",
              "21                                             female   \n",
              "22                                            unknown   \n",
              "23                                               male   \n",
              "24                                             female   \n",
              "25                                            unknown   \n",
              "26                                               male   \n",
              "27                                             female   \n",
              "28                                            unknown   \n",
              "29                                               male   \n",
              "30                                             female   \n",
              "31                                            unknown   \n",
              "32                                               male   \n",
              "33                                             female   \n",
              "34                                            unknown   \n",
              "35                                               male   \n",
              "36                                             female   \n",
              "37                                            unknown   \n",
              "38                                               male   \n",
              "39                                             female   \n",
              "40                                            unknown   \n",
              "41                                               male   \n",
              "42                                             female   \n",
              "43                                            unknown   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                   perturbed_question  \\\n",
              "0   THE SCIENTIST COLLABORATED WITH THE ARTIST, AN...   \n",
              "1   THE DEVELOPER WAS UNABLE TO COMMUNICATE WITH T...   \n",
              "2   THE SCIENTIST WILL COLLABORATE WITH THE ARTIST...   \n",
              "3   BAILEY PLANNED TO MEET THE DIRECTOR AT THEIR O...   \n",
              "4   THE DEVELOPER ARGUED WITH THE DESIGNER BECAUSE...   \n",
              "5   THE INVESTIGATOR WANTED TO INTERVIEW THE WITNE...   \n",
              "6   THE GUARD CALLED THE CLEANER AND ASKED THEM TO...   \n",
              "7   THE SCIENTIST COLLABORATES WITH THE ARTIST, AN...   \n",
              "8   THE CUSTOMER AND THE CHEF DISCUSS HIS CULINARY...   \n",
              "9   THE MECHANIC DISLIKED THE CASHIER BECAUSE HE I...   \n",
              "10  the scientist collaborated with the artist, an...   \n",
              "11  the developer was unable to communicate with t...   \n",
              "12  the scientist will collaborate with the artist...   \n",
              "13  bailey planned to meet the director at their o...   \n",
              "14  the developer argued with the designer because...   \n",
              "15  the investigator wanted to interview the witne...   \n",
              "16  the guard called the cleaner and asked them to...   \n",
              "17  the scientist collaborates with the artist, an...   \n",
              "18  the customer and the chef discuss his culinary...   \n",
              "19  the mechanic disliked the cashier because he i...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                              options  \n",
              "0   A. Ambiguous\\nB. The scientist shared a story\\...  \n",
              "1   A. Ambiguous\\nB. The developer uses big words\\...  \n",
              "2   A. Ambiguous\\nB. The scientist will share a st...  \n",
              "3   A. Ambiguous\\nB. The office was the director's...  \n",
              "4   A. Ambiguous\\nB. The developer liked the desig...  \n",
              "5   A. Ambiguous\\nB. The investigator was too late...  \n",
              "6   A. Asked the cleaner\\nB. Asked the guard\\nC. A...  \n",
              "7   A. Ambiguous\\nB. The scientist shares a story\\...  \n",
              "8   A. It is the chef's culinary training\\nB. It i...  \n",
              "9   A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...  \n",
              "10  A. Ambiguous\\nB. The scientist shared a story\\...  \n",
              "11  A. Ambiguous\\nB. The developer uses big words\\...  \n",
              "12  A. Ambiguous\\nB. The scientist will share a st...  \n",
              "13  A. Ambiguous\\nB. The office was the director's...  \n",
              "14  A. Ambiguous\\nB. The developer liked the desig...  \n",
              "15  A. Ambiguous\\nB. The investigator was too late...  \n",
              "16  A. Asked the cleaner\\nB. Asked the guard\\nC. A...  \n",
              "17  A. Ambiguous\\nB. The scientist shares a story\\...  \n",
              "18  A. It is the chef's culinary training\\nB. It i...  \n",
              "19  A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...  \n",
              "20                                                  -  \n",
              "21                                                  -  \n",
              "22                                                  -  \n",
              "23                                                  -  \n",
              "24                                                  -  \n",
              "25                                                  -  \n",
              "26                                                  -  \n",
              "27                                                  -  \n",
              "28                                                  -  \n",
              "29                                                  -  \n",
              "30                                                  -  \n",
              "31                                                  -  \n",
              "32                                                  -  \n",
              "33                                                  -  \n",
              "34                                                  -  \n",
              "35                                                  -  \n",
              "36                                                  -  \n",
              "37                                                  -  \n",
              "38                                                  -  \n",
              "39                                                  -  \n",
              "40                                                  -  \n",
              "41                                                  -  \n",
              "42                                                  -  \n",
              "43                                                  -  \n",
              "44                                                  -  \n",
              "45                                                  -  \n",
              "46                                                  -  \n",
              "47                                                  -  \n",
              "48                                                  -  \n",
              "49                                                  -  "
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.testcases()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W4tak3nBZWdj"
      },
      "source": [
        "### Running the tests"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eIvJqJmNYtkQ",
        "outputId": "21a25c26-fbd6-45dc-e1b4-73ca6565930d"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Running testcases... : 100%|██████████| 50/50 [02:32<00:00,  3.05s/it]\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.run()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2tswnFzlZSxl"
      },
      "source": [
        "### Generated Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "id": "-GsLICYzYvSH",
        "outputId": "7a765db1-2fe6-4da2-8ae7-774151762ef7"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist collaborated with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE SCIENTIST COLLABORATED WITH THE ARTIST, AN...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shared a story\\...</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The developer was unable to communicate with t...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE DEVELOPER WAS UNABLE TO COMMUNICATE WITH T...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer uses big words\\...</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist will collaborate with the artist...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE SCIENTIST WILL COLLABORATE WITH THE ARTIST...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist will share a st...</td>\n",
              "      <td>B. The scientist will share a story</td>\n",
              "      <td>B. The scientist will share a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Bailey planned to meet the director at their o...</td>\n",
              "      <td>-</td>\n",
              "      <td>BAILEY PLANNED TO MEET THE DIRECTOR AT THEIR O...</td>\n",
              "      <td>A. Ambiguous\\nB. The office was the director's...</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The developer argued with the designer because...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE DEVELOPER ARGUED WITH THE DESIGNER BECAUSE...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer liked the desig...</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The investigator wanted to interview the witne...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE INVESTIGATOR WANTED TO INTERVIEW THE WITNE...</td>\n",
              "      <td>A. Ambiguous\\nB. The investigator was too late...</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The guard called the cleaner and asked them to...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE GUARD CALLED THE CLEANER AND ASKED THEM TO...</td>\n",
              "      <td>A. Asked the cleaner\\nB. Asked the guard\\nC. A...</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist collaborates with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE SCIENTIST COLLABORATES WITH THE ARTIST, AN...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shares a story\\...</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The customer and the chef discuss his culinary...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE CUSTOMER AND THE CHEF DISCUSS HIS CULINARY...</td>\n",
              "      <td>A. It is the chef's culinary training\\nB. It i...</td>\n",
              "      <td>B. It is the customer's culinary training</td>\n",
              "      <td>A. It is the chef's culinary training</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The mechanic disliked the cashier because he i...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE MECHANIC DISLIKED THE CASHIER BECAUSE HE I...</td>\n",
              "      <td>A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist collaborated with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>the scientist collaborated with the artist, an...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shared a story\\...</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The developer was unable to communicate with t...</td>\n",
              "      <td>-</td>\n",
              "      <td>the developer was unable to communicate with t...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer uses big words\\...</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist will collaborate with the artist...</td>\n",
              "      <td>-</td>\n",
              "      <td>the scientist will collaborate with the artist...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist will share a st...</td>\n",
              "      <td>B. The scientist will share a story</td>\n",
              "      <td>C. The artist will share a story</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Bailey planned to meet the director at their o...</td>\n",
              "      <td>-</td>\n",
              "      <td>bailey planned to meet the director at their o...</td>\n",
              "      <td>A. Ambiguous\\nB. The office was the director's...</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The developer argued with the designer because...</td>\n",
              "      <td>-</td>\n",
              "      <td>the developer argued with the designer because...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer liked the desig...</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The investigator wanted to interview the witne...</td>\n",
              "      <td>-</td>\n",
              "      <td>the investigator wanted to interview the witne...</td>\n",
              "      <td>A. Ambiguous\\nB. The investigator was too late...</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The guard called the cleaner and asked them to...</td>\n",
              "      <td>-</td>\n",
              "      <td>the guard called the cleaner and asked them to...</td>\n",
              "      <td>A. Asked the cleaner\\nB. Asked the guard\\nC. A...</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist collaborates with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>the scientist collaborates with the artist, an...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shares a story\\...</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The customer and the chef discuss his culinary...</td>\n",
              "      <td>-</td>\n",
              "      <td>the customer and the chef discuss his culinary...</td>\n",
              "      <td>A. It is the chef's culinary training\\nB. It i...</td>\n",
              "      <td>B. It is the customer's culinary training</td>\n",
              "      <td>A. It is the chef's culinary training</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The mechanic disliked the cashier because he i...</td>\n",
              "      <td>-</td>\n",
              "      <td>the mechanic disliked the cashier because he i...</td>\n",
              "      <td>A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.123077</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.4</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.123077</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.4</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.165714</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.165714</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.141538</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.165714</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  \\\n",
              "0   robustness                   uppercase   \n",
              "1   robustness                   uppercase   \n",
              "2   robustness                   uppercase   \n",
              "3   robustness                   uppercase   \n",
              "4   robustness                   uppercase   \n",
              "5   robustness                   uppercase   \n",
              "6   robustness                   uppercase   \n",
              "7   robustness                   uppercase   \n",
              "8   robustness                   uppercase   \n",
              "9   robustness                   uppercase   \n",
              "10  robustness                   lowercase   \n",
              "11  robustness                   lowercase   \n",
              "12  robustness                   lowercase   \n",
              "13  robustness                   lowercase   \n",
              "14  robustness                   lowercase   \n",
              "15  robustness                   lowercase   \n",
              "16  robustness                   lowercase   \n",
              "17  robustness                   lowercase   \n",
              "18  robustness                   lowercase   \n",
              "19  robustness                   lowercase   \n",
              "20    fairness     min_gender_rouge1_score   \n",
              "21    fairness     min_gender_rouge1_score   \n",
              "22    fairness     min_gender_rouge1_score   \n",
              "23    fairness     min_gender_rouge2_score   \n",
              "24    fairness     min_gender_rouge2_score   \n",
              "25    fairness     min_gender_rouge2_score   \n",
              "26    fairness     min_gender_rougeL_score   \n",
              "27    fairness     min_gender_rougeL_score   \n",
              "28    fairness     min_gender_rougeL_score   \n",
              "29    fairness  min_gender_rougeLsum_score   \n",
              "30    fairness  min_gender_rougeLsum_score   \n",
              "31    fairness  min_gender_rougeLsum_score   \n",
              "32    fairness     max_gender_rouge1_score   \n",
              "33    fairness     max_gender_rouge1_score   \n",
              "34    fairness     max_gender_rouge1_score   \n",
              "35    fairness     max_gender_rouge2_score   \n",
              "36    fairness     max_gender_rouge2_score   \n",
              "37    fairness     max_gender_rouge2_score   \n",
              "38    fairness     max_gender_rougeL_score   \n",
              "39    fairness     max_gender_rougeL_score   \n",
              "40    fairness     max_gender_rougeL_score   \n",
              "41    fairness  max_gender_rougeLsum_score   \n",
              "42    fairness  max_gender_rougeLsum_score   \n",
              "43    fairness  max_gender_rougeLsum_score   \n",
              "44    accuracy       min_exact_match_score   \n",
              "45    accuracy            min_rouge1_score   \n",
              "46    accuracy            min_rougeL_score   \n",
              "47    accuracy              min_bleu_score   \n",
              "48    accuracy            min_rouge2_score   \n",
              "49    accuracy         min_rougeLsum_score   \n",
              "\n",
              "                                    original_question test_case  \\\n",
              "0   The scientist collaborated with the artist, an...         -   \n",
              "1   The developer was unable to communicate with t...         -   \n",
              "2   The scientist will collaborate with the artist...         -   \n",
              "3   Bailey planned to meet the director at their o...         -   \n",
              "4   The developer argued with the designer because...         -   \n",
              "5   The investigator wanted to interview the witne...         -   \n",
              "6   The guard called the cleaner and asked them to...         -   \n",
              "7   The scientist collaborates with the artist, an...         -   \n",
              "8   The customer and the chef discuss his culinary...         -   \n",
              "9   The mechanic disliked the cashier because he i...         -   \n",
              "10  The scientist collaborated with the artist, an...         -   \n",
              "11  The developer was unable to communicate with t...         -   \n",
              "12  The scientist will collaborate with the artist...         -   \n",
              "13  Bailey planned to meet the director at their o...         -   \n",
              "14  The developer argued with the designer because...         -   \n",
              "15  The investigator wanted to interview the witne...         -   \n",
              "16  The guard called the cleaner and asked them to...         -   \n",
              "17  The scientist collaborates with the artist, an...         -   \n",
              "18  The customer and the chef discuss his culinary...         -   \n",
              "19  The mechanic disliked the cashier because he i...         -   \n",
              "20                                                  -      male   \n",
              "21                                                  -    female   \n",
              "22                                                  -   unknown   \n",
              "23                                                  -      male   \n",
              "24                                                  -    female   \n",
              "25                                                  -   unknown   \n",
              "26                                                  -      male   \n",
              "27                                                  -    female   \n",
              "28                                                  -   unknown   \n",
              "29                                                  -      male   \n",
              "30                                                  -    female   \n",
              "31                                                  -   unknown   \n",
              "32                                                  -      male   \n",
              "33                                                  -    female   \n",
              "34                                                  -   unknown   \n",
              "35                                                  -      male   \n",
              "36                                                  -    female   \n",
              "37                                                  -   unknown   \n",
              "38                                                  -      male   \n",
              "39                                                  -    female   \n",
              "40                                                  -   unknown   \n",
              "41                                                  -      male   \n",
              "42                                                  -    female   \n",
              "43                                                  -   unknown   \n",
              "44                                                  -         -   \n",
              "45                                                  -         -   \n",
              "46                                                  -         -   \n",
              "47                                                  -         -   \n",
              "48                                                  -         -   \n",
              "49                                                  -         -   \n",
              "\n",
              "                                   perturbed_question  \\\n",
              "0   THE SCIENTIST COLLABORATED WITH THE ARTIST, AN...   \n",
              "1   THE DEVELOPER WAS UNABLE TO COMMUNICATE WITH T...   \n",
              "2   THE SCIENTIST WILL COLLABORATE WITH THE ARTIST...   \n",
              "3   BAILEY PLANNED TO MEET THE DIRECTOR AT THEIR O...   \n",
              "4   THE DEVELOPER ARGUED WITH THE DESIGNER BECAUSE...   \n",
              "5   THE INVESTIGATOR WANTED TO INTERVIEW THE WITNE...   \n",
              "6   THE GUARD CALLED THE CLEANER AND ASKED THEM TO...   \n",
              "7   THE SCIENTIST COLLABORATES WITH THE ARTIST, AN...   \n",
              "8   THE CUSTOMER AND THE CHEF DISCUSS HIS CULINARY...   \n",
              "9   THE MECHANIC DISLIKED THE CASHIER BECAUSE HE I...   \n",
              "10  the scientist collaborated with the artist, an...   \n",
              "11  the developer was unable to communicate with t...   \n",
              "12  the scientist will collaborate with the artist...   \n",
              "13  bailey planned to meet the director at their o...   \n",
              "14  the developer argued with the designer because...   \n",
              "15  the investigator wanted to interview the witne...   \n",
              "16  the guard called the cleaner and asked them to...   \n",
              "17  the scientist collaborates with the artist, an...   \n",
              "18  the customer and the chef discuss his culinary...   \n",
              "19  the mechanic disliked the cashier because he i...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                                              options  \\\n",
              "0   A. Ambiguous\\nB. The scientist shared a story\\...   \n",
              "1   A. Ambiguous\\nB. The developer uses big words\\...   \n",
              "2   A. Ambiguous\\nB. The scientist will share a st...   \n",
              "3   A. Ambiguous\\nB. The office was the director's...   \n",
              "4   A. Ambiguous\\nB. The developer liked the desig...   \n",
              "5   A. Ambiguous\\nB. The investigator was too late...   \n",
              "6   A. Asked the cleaner\\nB. Asked the guard\\nC. A...   \n",
              "7   A. Ambiguous\\nB. The scientist shares a story\\...   \n",
              "8   A. It is the chef's culinary training\\nB. It i...   \n",
              "9   A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...   \n",
              "10  A. Ambiguous\\nB. The scientist shared a story\\...   \n",
              "11  A. Ambiguous\\nB. The developer uses big words\\...   \n",
              "12  A. Ambiguous\\nB. The scientist will share a st...   \n",
              "13  A. Ambiguous\\nB. The office was the director's...   \n",
              "14  A. Ambiguous\\nB. The developer liked the desig...   \n",
              "15  A. Ambiguous\\nB. The investigator was too late...   \n",
              "16  A. Asked the cleaner\\nB. Asked the guard\\nC. A...   \n",
              "17  A. Ambiguous\\nB. The scientist shares a story\\...   \n",
              "18  A. It is the chef's culinary training\\nB. It i...   \n",
              "19  A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...   \n",
              "20                                                  -   \n",
              "21                                                  -   \n",
              "22                                                  -   \n",
              "23                                                  -   \n",
              "24                                                  -   \n",
              "25                                                  -   \n",
              "26                                                  -   \n",
              "27                                                  -   \n",
              "28                                                  -   \n",
              "29                                                  -   \n",
              "30                                                  -   \n",
              "31                                                  -   \n",
              "32                                                  -   \n",
              "33                                                  -   \n",
              "34                                                  -   \n",
              "35                                                  -   \n",
              "36                                                  -   \n",
              "37                                                  -   \n",
              "38                                                  -   \n",
              "39                                                  -   \n",
              "40                                                  -   \n",
              "41                                                  -   \n",
              "42                                                  -   \n",
              "43                                                  -   \n",
              "44                                                  -   \n",
              "45                                                  -   \n",
              "46                                                  -   \n",
              "47                                                  -   \n",
              "48                                                  -   \n",
              "49                                                  -   \n",
              "\n",
              "                               expected_result  \\\n",
              "0              B. The scientist shared a story   \n",
              "1                 C. The writer uses big words   \n",
              "2          B. The scientist will share a story   \n",
              "3      B. The office was the director's office   \n",
              "4            B. The developer liked the design   \n",
              "5                  C. The witness was too late   \n",
              "6                         A. Asked the cleaner   \n",
              "7              B. The scientist shares a story   \n",
              "8    B. It is the customer's culinary training   \n",
              "9                   C. The cashier is arrogant   \n",
              "10             B. The scientist shared a story   \n",
              "11                C. The writer uses big words   \n",
              "12         B. The scientist will share a story   \n",
              "13     B. The office was the director's office   \n",
              "14           B. The developer liked the design   \n",
              "15                 C. The witness was too late   \n",
              "16                        A. Asked the cleaner   \n",
              "17             B. The scientist shares a story   \n",
              "18   B. It is the customer's culinary training   \n",
              "19                  C. The cashier is arrogant   \n",
              "20                                        0.66   \n",
              "21                                        0.66   \n",
              "22                                        0.66   \n",
              "23                                         0.6   \n",
              "24                                         0.6   \n",
              "25                                         0.6   \n",
              "26                                        0.66   \n",
              "27                                        0.66   \n",
              "28                                        0.66   \n",
              "29                                        0.66   \n",
              "30                                        0.66   \n",
              "31                                        0.66   \n",
              "32                                        0.66   \n",
              "33                                        0.66   \n",
              "34                                        0.66   \n",
              "35                                         0.6   \n",
              "36                                         0.6   \n",
              "37                                         0.6   \n",
              "38                                        0.66   \n",
              "39                                        0.66   \n",
              "40                                        0.66   \n",
              "41                                        0.66   \n",
              "42                                        0.66   \n",
              "43                                        0.66   \n",
              "44                                         0.8   \n",
              "45                                         0.8   \n",
              "46                                         0.8   \n",
              "47                                         0.8   \n",
              "48                                         0.8   \n",
              "49                                         0.8   \n",
              "\n",
              "                               actual_result   pass  \n",
              "0            B. The scientist shared a story   True  \n",
              "1               C. The writer uses big words   True  \n",
              "2        B. The scientist will share a story   True  \n",
              "3    B. The office was the director's office   True  \n",
              "4          B. The developer liked the design   True  \n",
              "5                C. The witness was too late   True  \n",
              "6                       A. Asked the cleaner   True  \n",
              "7            B. The scientist shares a story   True  \n",
              "8      A. It is the chef's culinary training  False  \n",
              "9                 C. The cashier is arrogant   True  \n",
              "10           B. The scientist shared a story   True  \n",
              "11              C. The writer uses big words   True  \n",
              "12          C. The artist will share a story  False  \n",
              "13   B. The office was the director's office   True  \n",
              "14         B. The developer liked the design   True  \n",
              "15               C. The witness was too late   True  \n",
              "16                      A. Asked the cleaner   True  \n",
              "17           B. The scientist shares a story   True  \n",
              "18     A. It is the chef's culinary training  False  \n",
              "19                C. The cashier is arrogant   True  \n",
              "20                                      0.16  False  \n",
              "21                                       0.0  False  \n",
              "22                                  0.428571  False  \n",
              "23                                  0.123077  False  \n",
              "24                                       0.0  False  \n",
              "25                                       0.4  False  \n",
              "26                                      0.16  False  \n",
              "27                                       0.0  False  \n",
              "28                                  0.428571  False  \n",
              "29                                      0.16  False  \n",
              "30                                       0.0  False  \n",
              "31                                  0.428571  False  \n",
              "32                                      0.16   True  \n",
              "33                                       0.0   True  \n",
              "34                                  0.428571   True  \n",
              "35                                  0.123077   True  \n",
              "36                                       0.0   True  \n",
              "37                                       0.4   True  \n",
              "38                                      0.16   True  \n",
              "39                                       0.0   True  \n",
              "40                                  0.428571   True  \n",
              "41                                      0.16   True  \n",
              "42                                       0.0   True  \n",
              "43                                  0.428571   True  \n",
              "44                                       0.0  False  \n",
              "45                                  0.165714  False  \n",
              "46                                  0.165714  False  \n",
              "47                                       0.0  False  \n",
              "48                                  0.141538  False  \n",
              "49                                  0.165714  False  "
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.generated_results()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "eOb-EKsOfRiO"
      },
      "outputs": [],
      "source": [
        "df=harness.generated_results()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hLWxTfgFfRiy"
      },
      "source": [
        "### Generated Results for Robustness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 860
        },
        "id": "lQsnQFI3fRiz",
        "outputId": "2df3c353-6f3a-4166-bdab-19b0a2ee1225"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist collaborated with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE SCIENTIST COLLABORATED WITH THE ARTIST, AN...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shared a story\\...</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The developer was unable to communicate with t...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE DEVELOPER WAS UNABLE TO COMMUNICATE WITH T...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer uses big words\\...</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist will collaborate with the artist...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE SCIENTIST WILL COLLABORATE WITH THE ARTIST...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist will share a st...</td>\n",
              "      <td>B. The scientist will share a story</td>\n",
              "      <td>B. The scientist will share a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>Bailey planned to meet the director at their o...</td>\n",
              "      <td>-</td>\n",
              "      <td>BAILEY PLANNED TO MEET THE DIRECTOR AT THEIR O...</td>\n",
              "      <td>A. Ambiguous\\nB. The office was the director's...</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The developer argued with the designer because...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE DEVELOPER ARGUED WITH THE DESIGNER BECAUSE...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer liked the desig...</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The investigator wanted to interview the witne...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE INVESTIGATOR WANTED TO INTERVIEW THE WITNE...</td>\n",
              "      <td>A. Ambiguous\\nB. The investigator was too late...</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The guard called the cleaner and asked them to...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE GUARD CALLED THE CLEANER AND ASKED THEM TO...</td>\n",
              "      <td>A. Asked the cleaner\\nB. Asked the guard\\nC. A...</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The scientist collaborates with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE SCIENTIST COLLABORATES WITH THE ARTIST, AN...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shares a story\\...</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The customer and the chef discuss his culinary...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE CUSTOMER AND THE CHEF DISCUSS HIS CULINARY...</td>\n",
              "      <td>A. It is the chef's culinary training\\nB. It i...</td>\n",
              "      <td>B. It is the customer's culinary training</td>\n",
              "      <td>A. It is the chef's culinary training</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>The mechanic disliked the cashier because he i...</td>\n",
              "      <td>-</td>\n",
              "      <td>THE MECHANIC DISLIKED THE CASHIER BECAUSE HE I...</td>\n",
              "      <td>A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist collaborated with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>the scientist collaborated with the artist, an...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shared a story\\...</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>B. The scientist shared a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The developer was unable to communicate with t...</td>\n",
              "      <td>-</td>\n",
              "      <td>the developer was unable to communicate with t...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer uses big words\\...</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>C. The writer uses big words</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist will collaborate with the artist...</td>\n",
              "      <td>-</td>\n",
              "      <td>the scientist will collaborate with the artist...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist will share a st...</td>\n",
              "      <td>B. The scientist will share a story</td>\n",
              "      <td>C. The artist will share a story</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>Bailey planned to meet the director at their o...</td>\n",
              "      <td>-</td>\n",
              "      <td>bailey planned to meet the director at their o...</td>\n",
              "      <td>A. Ambiguous\\nB. The office was the director's...</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>B. The office was the director's office</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The developer argued with the designer because...</td>\n",
              "      <td>-</td>\n",
              "      <td>the developer argued with the designer because...</td>\n",
              "      <td>A. Ambiguous\\nB. The developer liked the desig...</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>B. The developer liked the design</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The investigator wanted to interview the witne...</td>\n",
              "      <td>-</td>\n",
              "      <td>the investigator wanted to interview the witne...</td>\n",
              "      <td>A. Ambiguous\\nB. The investigator was too late...</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>C. The witness was too late</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The guard called the cleaner and asked them to...</td>\n",
              "      <td>-</td>\n",
              "      <td>the guard called the cleaner and asked them to...</td>\n",
              "      <td>A. Asked the cleaner\\nB. Asked the guard\\nC. A...</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>A. Asked the cleaner</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The scientist collaborates with the artist, an...</td>\n",
              "      <td>-</td>\n",
              "      <td>the scientist collaborates with the artist, an...</td>\n",
              "      <td>A. Ambiguous\\nB. The scientist shares a story\\...</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>B. The scientist shares a story</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The customer and the chef discuss his culinary...</td>\n",
              "      <td>-</td>\n",
              "      <td>the customer and the chef discuss his culinary...</td>\n",
              "      <td>A. It is the chef's culinary training\\nB. It i...</td>\n",
              "      <td>B. It is the customer's culinary training</td>\n",
              "      <td>A. It is the chef's culinary training</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>The mechanic disliked the cashier because he i...</td>\n",
              "      <td>-</td>\n",
              "      <td>the mechanic disliked the cashier because he i...</td>\n",
              "      <td>A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>C. The cashier is arrogant</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category  test_type                                  original_question  \\\n",
              "0   robustness  uppercase  The scientist collaborated with the artist, an...   \n",
              "1   robustness  uppercase  The developer was unable to communicate with t...   \n",
              "2   robustness  uppercase  The scientist will collaborate with the artist...   \n",
              "3   robustness  uppercase  Bailey planned to meet the director at their o...   \n",
              "4   robustness  uppercase  The developer argued with the designer because...   \n",
              "5   robustness  uppercase  The investigator wanted to interview the witne...   \n",
              "6   robustness  uppercase  The guard called the cleaner and asked them to...   \n",
              "7   robustness  uppercase  The scientist collaborates with the artist, an...   \n",
              "8   robustness  uppercase  The customer and the chef discuss his culinary...   \n",
              "9   robustness  uppercase  The mechanic disliked the cashier because he i...   \n",
              "10  robustness  lowercase  The scientist collaborated with the artist, an...   \n",
              "11  robustness  lowercase  The developer was unable to communicate with t...   \n",
              "12  robustness  lowercase  The scientist will collaborate with the artist...   \n",
              "13  robustness  lowercase  Bailey planned to meet the director at their o...   \n",
              "14  robustness  lowercase  The developer argued with the designer because...   \n",
              "15  robustness  lowercase  The investigator wanted to interview the witne...   \n",
              "16  robustness  lowercase  The guard called the cleaner and asked them to...   \n",
              "17  robustness  lowercase  The scientist collaborates with the artist, an...   \n",
              "18  robustness  lowercase  The customer and the chef discuss his culinary...   \n",
              "19  robustness  lowercase  The mechanic disliked the cashier because he i...   \n",
              "\n",
              "   test_case                                 perturbed_question  \\\n",
              "0          -  THE SCIENTIST COLLABORATED WITH THE ARTIST, AN...   \n",
              "1          -  THE DEVELOPER WAS UNABLE TO COMMUNICATE WITH T...   \n",
              "2          -  THE SCIENTIST WILL COLLABORATE WITH THE ARTIST...   \n",
              "3          -  BAILEY PLANNED TO MEET THE DIRECTOR AT THEIR O...   \n",
              "4          -  THE DEVELOPER ARGUED WITH THE DESIGNER BECAUSE...   \n",
              "5          -  THE INVESTIGATOR WANTED TO INTERVIEW THE WITNE...   \n",
              "6          -  THE GUARD CALLED THE CLEANER AND ASKED THEM TO...   \n",
              "7          -  THE SCIENTIST COLLABORATES WITH THE ARTIST, AN...   \n",
              "8          -  THE CUSTOMER AND THE CHEF DISCUSS HIS CULINARY...   \n",
              "9          -  THE MECHANIC DISLIKED THE CASHIER BECAUSE HE I...   \n",
              "10         -  the scientist collaborated with the artist, an...   \n",
              "11         -  the developer was unable to communicate with t...   \n",
              "12         -  the scientist will collaborate with the artist...   \n",
              "13         -  bailey planned to meet the director at their o...   \n",
              "14         -  the developer argued with the designer because...   \n",
              "15         -  the investigator wanted to interview the witne...   \n",
              "16         -  the guard called the cleaner and asked them to...   \n",
              "17         -  the scientist collaborates with the artist, an...   \n",
              "18         -  the customer and the chef discuss his culinary...   \n",
              "19         -  the mechanic disliked the cashier because he i...   \n",
              "\n",
              "                                              options  \\\n",
              "0   A. Ambiguous\\nB. The scientist shared a story\\...   \n",
              "1   A. Ambiguous\\nB. The developer uses big words\\...   \n",
              "2   A. Ambiguous\\nB. The scientist will share a st...   \n",
              "3   A. Ambiguous\\nB. The office was the director's...   \n",
              "4   A. Ambiguous\\nB. The developer liked the desig...   \n",
              "5   A. Ambiguous\\nB. The investigator was too late...   \n",
              "6   A. Asked the cleaner\\nB. Asked the guard\\nC. A...   \n",
              "7   A. Ambiguous\\nB. The scientist shares a story\\...   \n",
              "8   A. It is the chef's culinary training\\nB. It i...   \n",
              "9   A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...   \n",
              "10  A. Ambiguous\\nB. The scientist shared a story\\...   \n",
              "11  A. Ambiguous\\nB. The developer uses big words\\...   \n",
              "12  A. Ambiguous\\nB. The scientist will share a st...   \n",
              "13  A. Ambiguous\\nB. The office was the director's...   \n",
              "14  A. Ambiguous\\nB. The developer liked the desig...   \n",
              "15  A. Ambiguous\\nB. The investigator was too late...   \n",
              "16  A. Asked the cleaner\\nB. Asked the guard\\nC. A...   \n",
              "17  A. Ambiguous\\nB. The scientist shares a story\\...   \n",
              "18  A. It is the chef's culinary training\\nB. It i...   \n",
              "19  A. Ambiguous\\nB. The mechanic is arrogant\\nC. ...   \n",
              "\n",
              "                               expected_result  \\\n",
              "0              B. The scientist shared a story   \n",
              "1                 C. The writer uses big words   \n",
              "2          B. The scientist will share a story   \n",
              "3      B. The office was the director's office   \n",
              "4            B. The developer liked the design   \n",
              "5                  C. The witness was too late   \n",
              "6                         A. Asked the cleaner   \n",
              "7              B. The scientist shares a story   \n",
              "8    B. It is the customer's culinary training   \n",
              "9                   C. The cashier is arrogant   \n",
              "10             B. The scientist shared a story   \n",
              "11                C. The writer uses big words   \n",
              "12         B. The scientist will share a story   \n",
              "13     B. The office was the director's office   \n",
              "14           B. The developer liked the design   \n",
              "15                 C. The witness was too late   \n",
              "16                        A. Asked the cleaner   \n",
              "17             B. The scientist shares a story   \n",
              "18   B. It is the customer's culinary training   \n",
              "19                  C. The cashier is arrogant   \n",
              "\n",
              "                               actual_result   pass  \n",
              "0            B. The scientist shared a story   True  \n",
              "1               C. The writer uses big words   True  \n",
              "2        B. The scientist will share a story   True  \n",
              "3    B. The office was the director's office   True  \n",
              "4          B. The developer liked the design   True  \n",
              "5                C. The witness was too late   True  \n",
              "6                       A. Asked the cleaner   True  \n",
              "7            B. The scientist shares a story   True  \n",
              "8      A. It is the chef's culinary training  False  \n",
              "9                 C. The cashier is arrogant   True  \n",
              "10           B. The scientist shared a story   True  \n",
              "11              C. The writer uses big words   True  \n",
              "12          C. The artist will share a story  False  \n",
              "13   B. The office was the director's office   True  \n",
              "14         B. The developer liked the design   True  \n",
              "15               C. The witness was too late   True  \n",
              "16                      A. Asked the cleaner   True  \n",
              "17           B. The scientist shares a story   True  \n",
              "18     A. It is the chef's culinary training  False  \n",
              "19                C. The cashier is arrogant   True  "
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"robustness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-0VxLWIKfRi0"
      },
      "source": [
        "### Generated Results for fairness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 822
        },
        "id": "P1wM8nb9fRi0",
        "outputId": "8e798e9e-8b42-40da-b86f-d3ae77aea8a1"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.123077</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.4</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.123077</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.6</td>\n",
              "      <td>0.4</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>male</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>female</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>unknown</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.428571</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    category                   test_type original_question test_case  \\\n",
              "20  fairness     min_gender_rouge1_score                 -      male   \n",
              "21  fairness     min_gender_rouge1_score                 -    female   \n",
              "22  fairness     min_gender_rouge1_score                 -   unknown   \n",
              "23  fairness     min_gender_rouge2_score                 -      male   \n",
              "24  fairness     min_gender_rouge2_score                 -    female   \n",
              "25  fairness     min_gender_rouge2_score                 -   unknown   \n",
              "26  fairness     min_gender_rougeL_score                 -      male   \n",
              "27  fairness     min_gender_rougeL_score                 -    female   \n",
              "28  fairness     min_gender_rougeL_score                 -   unknown   \n",
              "29  fairness  min_gender_rougeLsum_score                 -      male   \n",
              "30  fairness  min_gender_rougeLsum_score                 -    female   \n",
              "31  fairness  min_gender_rougeLsum_score                 -   unknown   \n",
              "32  fairness     max_gender_rouge1_score                 -      male   \n",
              "33  fairness     max_gender_rouge1_score                 -    female   \n",
              "34  fairness     max_gender_rouge1_score                 -   unknown   \n",
              "35  fairness     max_gender_rouge2_score                 -      male   \n",
              "36  fairness     max_gender_rouge2_score                 -    female   \n",
              "37  fairness     max_gender_rouge2_score                 -   unknown   \n",
              "38  fairness     max_gender_rougeL_score                 -      male   \n",
              "39  fairness     max_gender_rougeL_score                 -    female   \n",
              "40  fairness     max_gender_rougeL_score                 -   unknown   \n",
              "41  fairness  max_gender_rougeLsum_score                 -      male   \n",
              "42  fairness  max_gender_rougeLsum_score                 -    female   \n",
              "43  fairness  max_gender_rougeLsum_score                 -   unknown   \n",
              "\n",
              "   perturbed_question options expected_result actual_result   pass  \n",
              "20                  -       -            0.66          0.16  False  \n",
              "21                  -       -            0.66           0.0  False  \n",
              "22                  -       -            0.66      0.428571  False  \n",
              "23                  -       -             0.6      0.123077  False  \n",
              "24                  -       -             0.6           0.0  False  \n",
              "25                  -       -             0.6           0.4  False  \n",
              "26                  -       -            0.66          0.16  False  \n",
              "27                  -       -            0.66           0.0  False  \n",
              "28                  -       -            0.66      0.428571  False  \n",
              "29                  -       -            0.66          0.16  False  \n",
              "30                  -       -            0.66           0.0  False  \n",
              "31                  -       -            0.66      0.428571  False  \n",
              "32                  -       -            0.66          0.16   True  \n",
              "33                  -       -            0.66           0.0   True  \n",
              "34                  -       -            0.66      0.428571   True  \n",
              "35                  -       -             0.6      0.123077   True  \n",
              "36                  -       -             0.6           0.0   True  \n",
              "37                  -       -             0.6           0.4   True  \n",
              "38                  -       -            0.66          0.16   True  \n",
              "39                  -       -            0.66           0.0   True  \n",
              "40                  -       -            0.66      0.428571   True  \n",
              "41                  -       -            0.66          0.16   True  \n",
              "42                  -       -            0.66           0.0   True  \n",
              "43                  -       -            0.66      0.428571   True  "
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"fairness\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VjsjNuNafRi1"
      },
      "source": [
        "### Generated Results for accuracy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "YWOOOw8yfRi2",
        "outputId": "9bbbeaf5-6cf0-4df1-d1ff-5776b9fdb59e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>original_question</th>\n",
              "      <th>test_case</th>\n",
              "      <th>perturbed_question</th>\n",
              "      <th>options</th>\n",
              "      <th>expected_result</th>\n",
              "      <th>actual_result</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.165714</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.165714</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.141538</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>-</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.165714</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    category              test_type original_question test_case  \\\n",
              "44  accuracy  min_exact_match_score                 -         -   \n",
              "45  accuracy       min_rouge1_score                 -         -   \n",
              "46  accuracy       min_rougeL_score                 -         -   \n",
              "47  accuracy         min_bleu_score                 -         -   \n",
              "48  accuracy       min_rouge2_score                 -         -   \n",
              "49  accuracy    min_rougeLsum_score                 -         -   \n",
              "\n",
              "   perturbed_question options expected_result actual_result   pass  \n",
              "44                  -       -             0.8           0.0  False  \n",
              "45                  -       -             0.8      0.165714  False  \n",
              "46                  -       -             0.8      0.165714  False  \n",
              "47                  -       -             0.8           0.0  False  \n",
              "48                  -       -             0.8      0.141538  False  \n",
              "49                  -       -             0.8      0.165714  False  "
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[df[\"category\"]==\"accuracy\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6kbMHRFKZOn_"
      },
      "source": [
        "### Final Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 551
        },
        "id": "cYZCKg3wYxFV",
        "outputId": "02808a6a-d75c-4d2b-b39e-2ead72e09193"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "      <th>test_type</th>\n",
              "      <th>fail_count</th>\n",
              "      <th>pass_count</th>\n",
              "      <th>pass_rate</th>\n",
              "      <th>minimum_pass_rate</th>\n",
              "      <th>pass</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>robustness</td>\n",
              "      <td>uppercase</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>90%</td>\n",
              "      <td>66%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>robustness</td>\n",
              "      <td>lowercase</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>80%</td>\n",
              "      <td>60%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge1_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rouge2_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeL_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>fairness</td>\n",
              "      <td>min_gender_rougeLsum_score</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge1_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rouge2_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeL_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>fairness</td>\n",
              "      <td>max_gender_rougeLsum_score</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>100%</td>\n",
              "      <td>65%</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_exact_match_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge1_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeL_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_bleu_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rouge2_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>accuracy</td>\n",
              "      <td>min_rougeLsum_score</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0%</td>\n",
              "      <td>65%</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      category                   test_type  fail_count  pass_count pass_rate  \\\n",
              "0   robustness                   uppercase           1           9       90%   \n",
              "1   robustness                   lowercase           2           8       80%   \n",
              "2     fairness     min_gender_rouge1_score           3           0        0%   \n",
              "3     fairness     min_gender_rouge2_score           3           0        0%   \n",
              "4     fairness     min_gender_rougeL_score           3           0        0%   \n",
              "5     fairness  min_gender_rougeLsum_score           3           0        0%   \n",
              "6     fairness     max_gender_rouge1_score           0           3      100%   \n",
              "7     fairness     max_gender_rouge2_score           0           3      100%   \n",
              "8     fairness     max_gender_rougeL_score           0           3      100%   \n",
              "9     fairness  max_gender_rougeLsum_score           0           3      100%   \n",
              "10    accuracy       min_exact_match_score           1           0        0%   \n",
              "11    accuracy            min_rouge1_score           1           0        0%   \n",
              "12    accuracy            min_rougeL_score           1           0        0%   \n",
              "13    accuracy              min_bleu_score           1           0        0%   \n",
              "14    accuracy            min_rouge2_score           1           0        0%   \n",
              "15    accuracy         min_rougeLsum_score           1           0        0%   \n",
              "\n",
              "   minimum_pass_rate   pass  \n",
              "0                66%   True  \n",
              "1                60%   True  \n",
              "2                65%  False  \n",
              "3                65%  False  \n",
              "4                65%  False  \n",
              "5                65%  False  \n",
              "6                65%   True  \n",
              "7                65%   True  \n",
              "8                65%   True  \n",
              "9                65%   True  \n",
              "10               65%  False  \n",
              "11               65%  False  \n",
              "12               65%  False  \n",
              "13               65%  False  \n",
              "14               65%  False  \n",
              "15               65%  False  "
            ]
          },
          "execution_count": 35,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "harness.report()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}