demo/tutorials/llm_notebooks/dataset-notebooks/MultiLexSum_dataset.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"-euMnuisAIDX"},"source":["![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUgAAABcCAYAAAAMJCwKAAAgAElEQVR4nOy9f5gcZ3Xn+znnra5pjcfKZCyNfqDIQgghZMdxZMfGxpbbwhjM2g4h2Ak/Nol3Aw5xEsLu5eHh8vCofNl9uFluLhiwhUi4zib3ZomcZBMgARsjt4RxbGIritcSsiyE0GpleSQLMYxHPd1V59w/qnq6Z6ZnNJJG/Ej6+zw9PW911fueeqvq1Pn9CucASZJokkzZaudirC666KKLcwWZ+y4TveyWJeW4/lKZYYD5mI2m8+YdH61Wk3Tux+uiiy66ODeYYwaZaKUysNSI7xSVtfj4MCPi9t8WLhzY+sADt9fndswuuuiii3ODaO66ShQSM7lvvYj8B6A8/pMIiM4/evToTuDI3I3ZRRdddHHuMIcMMocgC9ysFwx3DBzVyFzCQBpF8VyP10UXXXRxrjDnDBJygdFyl4wiTS3egJPnYrguuuiii3MCPRedem57NHBk3A6pwLxzMVwXXXTRxTnBnEmQSZJ/xP2gaDjhrv00vTSigB12tVqSJNrcf/p+uiFBXXTRxY8ec+7Fvuqq+f1RT/ktgl40PogwbKn/XQgv7KhUsJwBJjNIr10G2UUXXfzocU7iICsV9AfnL4k5nG85//zYKpXv1pMksStv+uT8eKy0RtyWqU9U8U1cU5e9Mb17qtU7anNPWxdddNHF7HEOGOTUTJpKBa1UsC271kYLjh79zyL6bnefP3F4b5JzxLEPvrhw4Z/v7sZMdtFFFz9CnBMGORW5On1V5YLVsUT/CNJrlnXcUzXg+JfU7c5K5ehQ1x7ZRRdd/KhwTsJ8JqMpTW7dzlJc+swykBZ3HpcdAfcMkVAGLVerKHl8UBdddNHFDx3nJMxn2sHMFYrEmrbtPyQxtosuuujitPBDlSDXbwgqDo4grUTtCRJkF1100cWPC+aIQc4uZMdMLAhtzDH/lo7KdhdddNHFjxZzwCATXbuWCNZO8/sWBgdfUvhuCh75hN8mM8P2djfKp4suuvjR4iwYZKLXvq7/YrGeD7jbIBxF3NskyZZ/JTc9LkyBBdP5XNxBwETV8OwwcKJSwarVM6ewiy666OJscEb6bJIkWq0uXOkS/ptqaZ1ZSqsoxQxwU/f28J7Jxzil6LwnG/aDD2zf+rtbz4S2Lrrooou5whlLkCa+LmjP8ix9KXUkEloWxBm+TaTwnDsmok+L6iHcIxcxaBzP0h98bnvlxe1szetLnu0JdtFFF12cKc6YQbprjLgiolKECzXlwVN9Fz2kmdumyPyhNLhGmRhEI9XqnceongFzLIpg0A0s76KLLuYILQaZJAobIZFZMphsgnQ4W7g7ICaAqp2oXHfs4K5dREePthsnZ2BySdPOWS2+K5bTvLG5rcsgu+iiizlBziCTRyIWDpY5ursO5PnPic8QunM3ofgvZ46T2eSp2tB04iRJYkmSpDOmFCau44x77e6II3GZ0s+U0bEyvq+PTc/2Ic8tw5fGJL5l9ky+iy666GJ65AxyydJVuN7OYh/lM88OIQwjz42QygjKMJ6OYlajhzqhd5Q7qFPJO/Ai7Lv5fx7VOHO7CfdZZPJsPtwLe9fxmb2D4H286IuJWYTqAvS8BbgsRmwAGCTL9gFb5mhuuuiii3/lyBlkqsuZN+8OsvogIaqhOgqhRikbJUtHca2TpaM0pE5afzBJNn5m/bb7VGkP8p74/3TtcSapBhODIjvDvj9I+fy7kbCGtF7GrBfPYtwUc8vXd3AIEdC5AEYXXXTRxZkgZ5Alt9yg6BH1sX5gfsHbNOdnriBQ7jVOvpRWqH72rHVYY3bGSytFNBqLkXSQrFFInN70hBffbmiYZYdddNFFF7NDIUECJcgZjytNxtiEA7iRpYqQTu2mubPMsi2AIGKz5LMCmOKmHeMtu3yxiy66OAeI2v6eIthbirVlRGGyq3imlMHJ7bbM60ICzMuatSrsTlmXRrFZqeNddNFFF3OIXEXtIBNOz5CauvfZQ0TqANXqRH47qyK5XYbZRRddnGNMlCDbMUWY7MyR2r3Ys4XjiKC4r61UPnMQsrJpi0lm+olDpfTE4Wo16cS6p6Gviy666GJuMZE1+mTD4/RcyFWsGcRzOpCWAKogHzGyjwATdPbg8QF06d2Vyv2fn75WRbc0WhdddHFuMclJAy3GM7lG4xSHSwp5QLa7W3uwT4t1easHkem1cqHVrWMi0XIXeY9Qa/LHtmOno+cnH801wydt6wa9d9HFjwgdVOxTOVya8N2W1YdE4wXi2YxH5BFERidm5u75/sVPDmAZIEsta/QC9YnHdex9GhrPHJ2YVbH9HDCsRG+6aaCvWg29k3+pVDanlcrzx//lMMr2eW2d08SVMP+lnOuPEdoz485Vptnk7LvTHSdxhbvJ04anw91nXm+hSV87XaeYl4kqdrsXe4oGOy7iWZWKVbJtu2HwfZlnG8VZPC1RCuLgbgMg/ePVfMaHLAZpfakI5gBxTOvHSUzwHGrY0zHHczXWU08tKZ8YyX4f918uwt5VwAwipfF0tbrkvUmS/EQzyZwBJkYClSo6NFRELly0FtjNll1Q1P+05vz/JJ9vF2eARGxqrYV2VIqaC8nE9ONT9lvUmWj2u2VXG9/bDbuHLO+bKf1Ob4OcUqpxIiOrVLAk+e2HIdl62WVLykuXTkfd8wCcGB78UAjRfzCrRyAzVBGapTR4jpjjbbdtiavVY+sybIUIRhaADIJHiB4DHprrMYeGxqK4HF6uIbrYLVMpXgiRBixr1EulenzKTn5skWilglarS/qvrty7LFTlNSby6gWLfJkg/Rw7rrB4FOG4kR1av97/6aGq7CXWw5VKcnxGR10Xs8Omb61A9l0OGXhQPv2tnfzOq/fOWf/JIxFLll2CPbsq3yCK6yj3f2c7d7z8xCmP37Ir5lhpGZEuxp5dCroAedl8JJQR78ElxTmJ7x0G389nnjuI7B0i8eP5+DMwysSVnzown/i5FaitI7rwSk74UpA+xFPcj7P0woPw3C42P/c0YfcBEj/R7HN6RuU+KS6yybgKKRVyzpwk9tRTjD711LQUKsC111nqba6Yyd7vZnvWPvEp9J09KpUkOjR8qC/WeXeKh7fnGToOLghR5GZPcg4Y5Lx5wTL31C2z3BSRM0jLR09H53rAHwKaUmC1urA3w25Q4ZYS4Ro3WyUiKqJ4YcMW0DyyIeBqtZLqARq+AwY/BTz+Iz2Rn2Q0JSd/7mpCuAejTKlkYB8C5oZBJolywZJBotIHSeVW8BSIEB2hkd4BfKHJJzof78rRby9nXvmjZI31CPNxi0GLpBAthCEDF0PCMCE6hNsOFu39Mg39exIfmZZJLn52HRq/DS29kbSxGhFFFEQUHBzDHUxSotJBTP+SZbs/1mSSE+MgRVpSZJP5TG5PqEp2ahWoZVcquivY38QCFq32KVleJ/rm0ATZM3aeQkCQCCd2J3aIEVVkJsn37CCtOyEPgZrgiPrJxBe/uKScuX44aM/HwX8NfBU47hlmDSyr5x+r45ZinoEQ46zGeKuJLYcfrsnjXxaaaqUoqhEiMVEMOoPD9ExQ0lVIuJjcfFYGIkLUj+hNwKn5hKS9qCwDGaD5rIWIfBGWDDzL81OiHiWEftzW4PZOeno/TmQbedm+pR2rj21+9hqi8iZEfhv31WgUIZr32RiDtFgJQRVEIpxVGOsIvdOo2DBVahxvnzkXShL42rai+0nGw9MNE+pM31w7aQzM8WbON27F2+aHgJ9873zTrnre+endIfT8dpaNxTiKoHnWapvtuWi3NRRxQ+WAethd9Ne1RZ4NJrAOn7uKqYkra3dHHLN1pPXlxeJTxRgZmN/A//vcfN75yuHpO7kb5J2FFJfm6cRwgKzxNwj/E6eGiaLWh6SvxFmPllbgBo2xBcQ9v0Wj3s/CAx8i8aFxO+aSfZcS9XycrL4OMyOUFLLDGF/CfRduI0BMlr4c90twW8d5fQsYPvY1vvuq4dxZNNmL3ZTOxnmYTGqfBQwIs+lqMmMYyw+cvEs7fXMNV/WiMlBLqJbTZ+b/SrFlF9HCkfR3Qii/O01PxiIStU+d5Kq1tiWdGoKKY/nLCEXYWS8xVKkkUdcOORdwxl/ycyk/vhAW0Ft+HZmVUVXS9CuUoktxHyREqxitryfxvwdmthU26z3kmtROTD7KC684NuWY+7/TT73+a2j0XsxXkDViSvHtZNn/4MIDnyHxlEXfHsDlA5hdipmhoY5nW8jC3bzn5QemjJ24sujAcn7w4luw7AtTnTQT4iCZJtJnbpjDqXtpqdo5q+yZ0OrYyU+usNUBk+M8f7JQLOi2lhDdlqVjfcJEdU5EUxE9CLbHPT3miKlIHxIGUF2M23KgTJb+c2znDXdXtpwrTHSyzgkSMe57bjlZdmmxxRC/n6h0F5ktQAOkfhNUv0Jy/Wm85DwizSKuQ0naH+674bsrhlny/B+TvZQSlT5CI+1HrZcQ3sBIbQtUh5CfWUccX06jDhqBsJVG9hGGXnFw2kLgL6w4SCL/9+TNp1Gs4sxQVAxXhe+rBMuQIrB8qoMGwAUTFBEZcer5pJ6qNNo5oHvSALPeczycZdK24vuslZvJ/Z+q79kEn7diECfHJZ4+vdUqmrpfEcxX57p06zeRAOJfERu7B0r76uXGcM+YGMRlPOuzLBuUwKVo6UqX8Pj1679bb94/pzqHs6F5ch/5N0yOx5yu/5lspDPRM/m4TmOeaozZn2+bdjgXKnYzHCYK1yC6ODdLZUOkPEpmr8eya8hSRaPXMPiy5SR+4LTjIrdhU45JNirPL6mx8MBfo+k7CKXX5GdkawjxAi5ccZyxxsWk9aW4QVwe4eTI3zH0qoP58dPQMA3j7BzmM9lDfJYe4yRJ7NprP/Gwp/V3hKh86cyKtqu51zJPv9DosSPAYO5JnkRnRw/73KEps+aUztx/O5NKinbTNzXl+5QPcbOo8ERUq2iSJIz3P8n5Nf3DO3176kOXKLPstxOSJNEvPzHQW66Fi9ysb9zmSG6gcLNhj/QDgeN7Ad5wVf6oVquMAMe2b0/23XbbliePHv3eFqE80hw3/y5oSzoO3U7EeJhFqyrU7BaBa55ra15a85Mk01/D6embpRNz/LgZmanl3uDmhsljnQpzrJWMMxq/CRUgMpxvsqh+jO/V/wcS1fAsJu5dRnbychLZf0rypqDDGlOJ5PNwdOMQS57bQ6nnNaR1cPqwrJ8fSMw8/Rncy+ApwgjoPujAbDuez0RMVLHbvdhNJjQeG3l2TOjrX//9pyuVe/+NWe0t7lZkjDTvvxZt4sFcbU9w2f7El39vhJvfNJinNLbR1ZG+uUXrwW6Xb6dWLE+SRLfsWhsNHj0yuH7Dp1bLtvCaRwivuA4WQBY/4jricOhasn/m2vt2fPnL6QFg+HSlnaEh9KuP9i+9Juu5YSty5XUbfCnmPLJN9nuWfSPL0scrleRwXhkp77dS2bQiwy/11FJVVVOxrdsye+3rP7Xz9a998UheZm7higy9/LrruQp0BdssAj3yCPbPlcq926vV3j1JktRnS2vISmURHURzb7XguIuJBpzs4Ne/dmRPMXPtqvN43xddtDtNkuRYs33ZZZt7zz+/foUZ860qputVATz69KEXLxh8ZvDobhsbmz9fe3rWbt2u16x3+XnB5rNBRrZW/cA1lU8+GNGzE5ITM9kyK5UkeuihRQPr19+76pFtevl118urcJaSe2VrW6scuZb0Wat86tFqNT5QqeT9VSr3l2H0cjMbaNJnKqbmCvcc2779vY91GqvOwou3bpPl11TMqIKuV0313oOPVe/aOXX/+8uZ1i6Rbb6Y9cWEVc2iikZZ+OTer3/t93af+so0X/fMnQ3yvj2X4H4NaUMRMdz/jtsvqrP52R2E6ABuq0nTAcRfxyef+wrHV00fjnMmj7Fbffx/kTpRGOWkKm5Riy+IgkzJUJstpqYaTpYUJ4f7nAWq1buOAPedar9WDF2HHzvSdy6NkNImQU50FiVJol/9av+yhfHRm116flHcLgcGkOZNEEAEcVdcUonCgbLKX1+74dN/Ua0e250kSZ0OaB9RALFQvmBwwVvUone523rRkN/iWkjiwm9GpWg7LL4HfusrkEuYW7dlG5Tojzx4DUHVzUTiUW003l+tLvxLM26UEL1PsHUQehGseY754pPRPhi9p1rt2wIc60DqjBhfkUhcPU9HXXbttYMXv+51Q8/kNHZUVydsmzcvW+we/YEIl6q4oYCLikd/0//9F38XLlhe6gn/HuRmcVla1CzNRxZXNfl3HvE3kl2wqVJJdnZikle94Y8HsrGxDaUe/SWMG9xYIKoTGEkeiqcaiR5w2Oos+KvLLttchXqvubwHid6q5PSpuEnQ2C3aWakkV7WPmSSJfvUbFwyW0ujDbtnNiqSIqASNStjDwE3ttFUqj0Rp2LU8ePRRd7+6SZO6mmsoq/EeYBYMsg1z5cVWuYFSOSIdM5BDYE8CUPf9SGMvImuwFOLyJdjoCrj7mbkZeCMs291PI1pNVoTqiB7ETx6j96U6dv4xJKQgkGXzwS7jwgMPkST1001TnL4e5GScczvfRJyWLekcO2m8k/yfJFqtXrA6RPGnIPrP4De4eb+54Vkzxq+BZ3XcU8AjsJUov68S3Zux4M1ffGpJOZfiOp9MMeWxpPZOJXwUZL27q2f1vN+sgWcNwMuOvxENH69U7nvNuBqdaU01KEgZJ0aIVUOs7ksz+A2Nev4Q/Grce90LWpv9muFuKyF8xCj/1k03fXL+bOIR43qtbm7H3a3wSkPLbCD9ov7Rr1YHr9iya+2kJYc7I4rE0JCiGmHEOLEEjZQwX+q22qV0r4j+O5ylbpm25iWPrQTvF5O3u0QfzbKB1ZP7r1TuXRzX7UMq0cfBf9VhgWOYNcav43if7ubmy8F/TSW+5/zz7feGFv70sKg+JSKG5/RhRSygyKpG44LBibdNYpr5MlFdKSqtawORO5dWKpsXTKRvm6mzGMIyEYnHx4AyeE1cpkioM6KIvT4rJIly/3f6gdcXy6AoIjtI64dJXHnx+SHcniCKR4EU95WIrJ05x7oN0wljSaLjtsK0VKHUs5YsNZAU9ypmx3j+sjruu4ii44hAWu8lKr2Z2tjVrL0tym2ns4+rzXecHObzI8aPX9zb1HmpVC9YnRE2icrNbul890wR0yYrLbJFtJ25upu6W+yZXy4e/vC8kcbNUyWacS++uhuOrBb0P7r7cstSLVxammcESB5bKK7uZu7Zmgzf+NBDixbkc+i1PI7eQUxx1KwRu8htKuH95o1lZinuZjjmbX2Cq3umjs8XLb3rByd1PcwmaPv7I0L2zyI6MjHeFXAzRG6MNHzugqGhjZXKp9aQd2rkJocpfTcaYybjBUscxNUtU7N0tbr/IcgVbhYVvNha8yKKgONq1oiRaL2WSu+f2HuirtHHReTd7tni/HwzBVcBXFAR1bbzUMSa46+QEH9w4dDQ73iWPSOqRxAMseJ6ZIjo/FJJV7aGK87RwnJ3W+qeX5e2/QfNGmsLm2lrPlJdhtsCt2J/DNEA5nvghT0zX49JmCsnTb1+MaXyGiw1oEaWfoOFHM+LSVyfYjwOHMctIksHiEpXMbCvb+blpAtMJ4s1+cLi564h6vkAWTqAqqL6NHbyAY4+MAoYFu3A/BmcCDMQ1hJKH+NY/MbChpnHSs6Clok7zCgl/ngwz444x8JtK+snI0kSrVQ2rXDCx1R0vecXILeL5a/nVELphIjsNfc9IcRDImEiE/RMRWWxEG2+9nX3XXLyZKaTw2HGz0noBe/L/1VUo1SQnKG17SqCmmdpFHpeE+L0LUmSqKnXJ3QoqHtWBrnULFuGmZL3aaKKeMs+JCKIiLplkWe2LEjpjmp14eBkp087kiSxSgUT9+2CPi46yd6UF0lWz7I1IcT/u0v0j9dtuO/Prq3c9+bXfnXJsi1b1kaTmWSppOZNHWe80ImD+EoRvcIsNQRVVUSDFT/bhIQrcfWsHrn7r61ff+/VkOhll23uXV8Z/AOV8KtZNtYLFo2fN2IaolGVsB9nt4TosGioC0W/goJFWVbrDaXeD6Csc2cvIupe3C3uphppBs0QGBLy1Etcf8GzbAGeL4ZXVLMy1aAeqOQ25MSqVbRaXdiL+s+6Zf15VpxAca+4yN9Xq0n6Q800ShKF65RM14MMgqRE8X5UHmf32nSciVn9ScZGnyaKQQKIVuixaSs2FCgW4ZMyJZayaPEyNn1rBfftXcnmZ9fw2b03sOQ7mwjRf8fSy9EIgj6O1d/LnWt35IxPjLtW7SPLPkb5vL2okku5cimBv+Wz+/8rn917Awt3D0JVT8UoO8dBdsT0XChx1yLwfE6QnKtyTKeBiT5yz62CrrlDRl+8WQjXFA/nuKoooiaqO71R36QavknGaCb1derhXaJhvVsWk8cwqVlmqqV+Se0DIZTeZ3gqjk728I8nZmrY75buMOe4qi4vJKeBPPOkuZdHZo35SrjuoccW/XUkmRVse1IuRe52EpW6oI+aNQ4gUtYQXeKWXTJZzc+7tyvAlkFy5NRe4Rf3Zb7gc0HjNe4sds90vB6ooI5hWcMQ6ROJ3i6kb45i/+bCRcf/qlod+AJwqOmpbzTESrGk3kZ38yxwN5HIVGSve7bTzU5I0NWIrMOy/lawQ26nVonVqN8CyWPnnffpimjp7WluP8sZjjuCGnAo8+xz5tnfSxSOq9sKcf6tiLzV3fpaHmGP0sbYAkF/CU+HNET1jCxu7w+4qDlfCfDahs0v9ZTWuhvuaZt06nlMs8vP33LL5t4vfvH5WrWKXX2j9pbSsAo3xX2cRvdsGPWvz3wXT4OzYqcb4WX7FuPhKtJ6nKuxjd00xiZ6qe+6aIRNzz6I6M1kYyC6CgmXksie6SvxCGCgcjla2gyhmTgQgffhtpigfWQpwGG88RUyPs6RVROl6MSVIzzEon0fpjzvD2iMrSgkXSPSd5Lpmyj1PsqSpV9G9lQ5fGR/EfIwTbmzM1GxN26EJOETu04ul2dH3+S/IhHuhoQzn37PDAKf+NWxR39/Tc/TZ9zPHKAV4tPGpAQbPHpk0CX+JfD5tN9qriYiJ9wb/3HDhmOPNjfv2rX20JEXXzyo5veAXOHuxUPratYwDfE1sTQuMbfc09tWetidIutEdpqnH80auj2ObbQRxgaiLHqnavR+t6y/RbXg5mgUrQhZulhdzCfFIgKIYwh1N/usRX5P5DIE9ahhsiYS+SOQi/OiGQV7dVPQxYJeDDyZJFPDh5oowmSoVuVLnjUGRMNHRaI+LyQ9mhlJuRqf21CFPjeviMrlaPn69Rs+/alq9dhjlQo0GuDixaJtE9ITTTQC829CfaNQ3yk6r4bbYkPuFA3vxrK+1jUS3DMQW1epbF7gkv0i7oMTcyDERMOwe/qpejn77BNfPj5S/HCgUhnYax56VUu3uzVyVb4ZDKa6yiwbVbeaIHFz3twzcF9dqfzU/GolGSZJrFTZNGDua5quxXH2KCi5mr36e99rLAP2QWKa3dcHvpKiDB5Cs97CHjLfe0axn2cjfiRibPrWKuKe1aR1I4pr1Eef4OjQMZKLWiXDAHTvw2SNEZBeNJSx7A3A508dD6n9aLSu+D9/EIpsXxr1lHweTiD+jwhD42M2+22mG76w6i9Z8u06qncRxVcDZRpjIKEfsVuReAORfpNFS/8W+/W/hOTI5MIas3fStIjPaSharqzE5f0CH0T0g4h/UNo+p9NG9QOi9gF3W3c6FJ17FGxSvJYSLnbzy3MnRpukpaqI/7Xasceq1evG4yIvumh3uviCC3YiPCAhGqG4PXMV1k1hIHO7HogmhDMB4KYhOu6SbQr0fimOXzherRwd/cbDJw6JN+7DssdEI9zb46QwdwZClg20r/Mz3qNDblPXrZbJPVE2dLBaPToK3x95fWXom5h/yt1TL9TUNptqZMgrZjNbuap9dHRkJPoTJ/tdYK+GWIubfeI5NhklmbpZn3t2q0rPPSkL3ghAb/uuzZNonoupB7sbjldh5ESlcnQUjh5Q5L+CPENbFXvH86ElLDUdW6caX+JmOm4eaaq41tiRxvqnN13ZZI5JEat5/DCBexxLc2bbJMrVzfpBBtzTWq5mA1DYFcNSiBZX8pU71Sxbi2XL3QxcwN3cyRMn3Ey1NKAlXdOkO8p8qbstd2tZs91NPfUdUDsx1ck3C5ypCJO4cv93yki4nLS+vAinOU4WHodKEaeZaDOPmedX78PZQVTKGZzZhsK5MzM8HSUdO0ha309aP0BaP0jWOIGIUe6NCAFCWM28+R/B5HMsfnbdxFqStOIan/+fX6KR3oll7ydLdxL1KFFJMQNPe0nTDcTzPkKJTWzad3F+bMtkMdFJMytPdfHMFXMgSorIqED+cUZo+0xoU7RpfSb9PuowKh3X3v7hYrKKXbzv64peJyrz80IWkjNJF3PLhh17II+N22btQc4PPLA7bbhvxX1IhOYDhLtoljV6Bb8cvJ/2cnCOiahmWX3Ig26tVr9br1aTwsaTWLX6vhMmfFk1dApk70uRPjWxKdIjmCg1cftiFA0drFQo+kvSJEksy6wqovtVWyFN7m6ImogOMkskSWK33PJ8bfsjd/1pGuQNZul/EtHdGnpG8WAgaev9InnxCnE1y2K37OJI40/Bomva+2wG0DuF9CiyY/vWux6qVpO0SX+lgp1/vu53T3eIaJ2mKNw80r2XNLrW8pTGCVCNMOVvH3voPUNF8HdxbP7/9q13PYbzpIQSTAjeFVWVsjsHRQPgzegzk1CanyKrxvcN4ToJIXYc1Qjwb6roweZS9OY+X+DSSmWccV+C+4LcOQOCpqLhmEn29Wrl+8OTVwSdHs2XPGcnQY6MDRDF16MaUeqBsZM7iE7sbDk/ig9AIinIA2SZkaVQ6lnOWHrD9J27FXRuh3Ataf3nSMd+lpPRzxHkZ2nUr4lUAr8AACAASURBVOXkS/8HIjuAlNEf9FMq3Uyp9//js/tvnVJkNxEjuT5l6JUHOLzyM8ThtaT1X6Y+9nlK8UE0GGZG/eR8gt5KpA+y6G2Xw8ZxJjnNu8QnqduT2y2IuYGnhtfBUnJ5tPPH2769rQ0pWNGWVPxUl3ASPefAf9SxSyNCfDWiJmBN+5yoIqqHTfwAdPbC+1jPQbf0cBFnaOMrO4orooOO9I+rn+MQBEZcs1pnlVYONetHTiyI45GgEaRtFq6m1wIDHcnwY3n17ok9RlGoC+SFSGWCGwiE0yrc25yHbzx858Ht1aGN4v4rno19VFQeEo0Oi2hK4RgaL3snglmmDstd+DCjcVSYGZjw2hJBjCPFSBPu48sue76myAtISPPzLc5B8nMQZRVu88enq/g2S8F9GtNOPoaITPrdEcFAyiqyF3dEirAmwRR6BVlRrWJr1xLltlyMgkE6uh2V/VLEznrWKLv5RbCkH8Al/KxoZDhWOHNURA+QsTe/dKeTauhn96wkYvREK/BsXe5gQlGG8f71fGbPGyd8Fu99I5959k14I8ZtBFFDxBC/iS27TnEfSUqqdY6uHeWui0Z438tP8K5XHuLoXzzO0OGP4GPvIEv/BNE6acOwdDUiG1my7JKOITxNafKOl9c48ud/g/a9i3r9DtLGnxLFJ9AI6jXQsJhS+WMs3bOqGZI0UcX2JuMZt8xPbY+jzSvj1BCpC1ITpCZyZh+EGlBDfHoJshN959SLPSFPPHZncOJdVgwucjzKQsfAb0isp+fQMHBMVWkvC+wO4tILEkNhMyzGbf2djjKvNfdoUz+104RMYbyGTX64kiTRRqTmkp9H03c/V2+gavWF3SLH/ou4v8fTsd8F+WNURmj6porxRFDPUhC9JoR0DWitKfw0YwUACFNfpM30wsyzurTJSs1XiLur4QvcPPY2ppFL9lkaEXUMiG97kRwZZw5FzwV6Ef8ndxsZZ+aOmmW94K+47JYl5YGBwWU4a1pFkQ1RnkD0ADC+sJ1GpeVZyJYmSaK4r83PurjOKlia7g2hdPA0pr5F55nGQTbVV/cKyCCWKY0xQ/RWouiPCD2fm/iJ/yj/lN6PWx9uSqMGGl/B96KVM4fYOJTHtPOyC9uMw2v2kcUfAdtCFEd5LCSXIvqOZsjYVPrb7J53Lh3lhVXbKcfvx+obCeEQGnImKXI5pu/gwgMxietEFRumMsJTqN2ipDmDo+ZCzdXqLlZ3L75ltm3qAjXwus2kBHSi7xxGII0/jrnEGkkeqNuyXTVvXJd6o6EdCysAVKuYIB0YqBgaVCZyiVlh5uq92Sn3mA06BsmfEZqmgSStVF44uGHDi19qjI1+yN3vEuFA4T0eH89xVKLY1K91UqWI5/TCwTPZMz89/cW3FDpsXso8br2AJrhL0jRk07zkmpCxcRW6SamBO+UU9uCyVzQycTcH3LNYkRXn/yCdLxGXiJb6MENENEsbdXWextLv5jZJDMHcWCoNX/zEE6v6EFbiha3U3VTDCGL/dGYLuZ3FszLOYPQNSGFL1qBEpQFgGSJLO390MSGKgNzuV4oW4375zI4agU5l9NvV96MrhsjsHiwbHY+Qc7uVe3f1zZgt01L/jRUHRvDz/gRr3IOEEUQhrZcpla9mNFsGc/AEpSmIWj2gGJh625uh+aKcZdudVHBcT9MGOUfPcLWKVSpphER9orlHeFzykkLddclVhZz28ZqGDr2lkk3jUUy0Urkwdk72NVlqy/nh6m41F6nLhBqJZ4hxlTLMvN8s0KJzbkX05hxVKsnw0MJlWwaODcVBo4+5Wb9IW9FVHHHWgMduTRUcaIsBPRXG59llvOakC3VEwFrsMZckJY4yZszbdbfzRbStXsr4CGnJ5TBBtnor9lFxjBAPYukCsNeqKJm4iUQK2d5K5ej+rdsu2Ccan3DL+t1dRWxQRFaMjIwckuCL3VtXwtyPoZxe9kzz/Jrc8UxtkPfuvRT8NWSN3K5kthfP9mAetdJrOw3tA2i4FKxMo94P0ev4+D99ie+fGMkXy/r26dHRYq5P80f7dhNK64qCFSuQsJIkyVMaT/UCuf76lOQRWPgzX6As/waXDQgpqsvRxjIS2TdRxT6ddMKNG4tDPBWRmkNNoO5IzZGaS/E5jTbqNReti4fTu4RzJEHmapSWaa7SKC0lU3Nj4xFROdQ+Ty0Hji2uYx09dEkCjdLIgIsvNjOgXfoUHDuheYXjlq3wNJhS59PPOM3whNPs/9Q4VQBztZqkg0d3W+S6WzU6RFtgeZ6P7gAxPiGb5bTombCvkJfTcx8SpD6+zEfBdTVEajbVeVOcSxF9wEpErKm+53lNggjHwWrm2T+4pXVENF9SRUxF+qGxGPe1ZllhRwSQJ5MkMXU9KKJDCCaCOl520VeGYKtVS3mWkGOiQS2r71Orn17udfPkzxYRNxKXI/KMpRouG3n+lb+Enn8bPaXpP0HuIpSeyV9KppTii+ntWwnbjLMNoHbJFwVzz71sQeaf4ohJqBiMHaFeP4Bqmj/O3otob37Krb9nhsjNTWuKmEEuR07Rfjrxu6nPjpF7XSU79xLkxLp/UKmgSZKk69dvWolk42EW446/nA8edOGo5OEhxc+Cu6mIDqpwCbBzciB1ksD6DaxRiRabp4wvN5BXuUnF0n2GRHqGrOicmmDPoP9OZdSa8zxRwk40l9qzMnh5siMwd1n5CYR+0dzHebr0tDQANHegaOruB1TCCcda0qKTB4wrVyVJ8qVOmkClcm+fua+T9vvZx42jB8BHXMMeNfYDa8wzlTy4e74RLhVhZV60Q3C31Mi+AZAGORwsPYSzGjBRAdFV7vYDFaWotI5IhEj69Wr1fSfOrIiwnNnNkiTKsn/fT+Pk68kaoAFE9yAndwDw/JJa5wML5jfwjv301J9Gw7p8jRlbidvFcN0cxDrnWWb5v2ago62c71nWg4t+2vAf1HKeZNY+SR1Y48RMjqntAm2MXyH1fGU6y4qU2BwtBaa1TSe1WxARyzNWbAYJshN9p4/JD0ClklCpJLr1Eb9LVPvNsjw+zwsmaKkiPEua7XMNI7j0uuQ5u7ntSGNxfxvwp8UImveLwoVRaiOvV2WBu1vTGC+CqZaGU8+eELefZ8JbY/bnNc0V4mwtKGf2LCVarS5a7mK3O/5MpXL/1mr1jmm88HDllQN9mcstkqYrEJ9EsIDotwS5zJuhQPlmbb+zZsbE2VEJqWm6C5FDIEvHexHUrAGU3vjwwwvur1SS/fnSxq2eTLhRJVpheXC7FhRansrOznovwyHzuro+jdvaptfZ3frEea2jA4ghqoAcDsiTAFHmQ+bZXtFSxTyFzFXUVpl5LJKNu/TMGmTIGdZXPxsv9kZo7LuEnvJqxk6ChgjsSYLlDq0Z6ywmyvFVIyx69h+Ie9/C2EvzcesnlK/ip1Z8gUsPjHB62eQth9GSvQO4ryJLc6btNkw9O3L65/eDXlwGsbQo2yajICMwOdVwfIXA5k0jrfY0T4umpRTSmqOWhzugrcfcaQmUxcbJAmZ72y0X1CSawYvdib7ZY+3aJB4cXHS1iS/1NN3nrieiKMRbt/pKUb9DVG81y3TcvuS5ucXhYObp0yX1Iy6lRxG/Ec8lcgTFUtMQ3bi+cu//1hjr+X96eg4VMWoLyyYnbw3S83bL0phchcpVJtHIspMHAjxs8PNeLHrkM7C8TpjgZsgdSLTbICevHHk6aB07OyRJYus33Ls60vPuzGxsmVntmfWVz2zH7B9V2Z8GhqJMLAvSGzJfaeLvwv1N7lY4UYq5QcnS2qiKPezwC+30nO55tJ+/4+oi+ywd+6ZoWGd56FbO7NxNlLUhkg/Coru3bHnhcJKQVqsXxnnNR/+ISRp5U5b1XMbVEO03sr+76crjI7t2ra0NHRv6Bwi34pTzQPJ0PrABsd7WlZKdwJE8E+aukfXXf/op1WjY0rQ/L4jhqwVZbtbIox60hFu2uyRHnzytk++E5vM203KsTSSee5Nl6XqcBagaGp2g0djG80PD8MDMYyWJkWxULNpO/eRhRPoRNczWMy9dyrZte1j0zkkHzeKhXvJ8GdffptSzgEbNiGIwHuPFVUdy73el5c2eaclZqkr2skvp6bmYRj1Pa/TsAMYhEtepSy6cUT1IrUsza2Py8ZM16RnahhgK0YTg3kk4i3qQuXTzU72m4VfE7TcJ0Ql1GTUhQhlAQtkss0lDGGAisr3k8QGIR8xH/0IlrMN1QdOp4DmTBJcPx3Hj1akt3HbttYxmLlep6O2epUvBtWlbaxaeyCz9XP1kOtRT1gjBcLS9HuRsMZVlZMW8hDNijNB8lGdPS5IkumULkWSsymx00N0jCdGlAusMUhOGg8mwo6mYlc19UDXEmRW1KNqcHqKKW/b5RoPDUezllg9b8NNw0sCkF4N7/gIJ/ldCuFHUV7lleYiNoG5ZJITbHR+8YHDwi1+r+rGgtVWWydtEdY2bjWsADiaqdcuyh+aVSzvzEKPd6QvbFz0j6BHwFYVwoUBuG3Mxx8zddo6OlIab8/a17faMWXZCkCKHXGKYGHcqKtXqI8k06uypZ2EqNkIyUzTARqCqLBlcisZXktbLedSF7CewO2dC15/aX5CIkTxygMVLHyOetzZP99OVqFxBkuxm0+3ka08V8OKZvo4iYHsjucpaqM6Lvr0Az94KelcRagRuJzC7H6rK4LLL0W/3k922k7suOjI1pKjoKxHj3r2XEOR3SRurwYxo3ijpS9tYYIcY6iRBTodpHDgaxtLM4xqSV0M5mzx4AcMhUzk9G+RpPC31uBzHKQs89zAOoDIghSrtZHnwdrPb3GZlInoos/pfBV48AZDFi/5eG/yChNJveFYvN1W+/CR8vov8RkDfCpK6WX9epqrlnRUXE1V1S78QGPt8Z4/zGbpG5Ix9lB26On0MDv5Ur6Gvxr0XUMtSy/3FROLaj0o/4uNOmMzSybdWKqqK2ZMe/F5ixnn9mUnAHc6jAcdeHHx84cKhTaLh4+QRNCYi6oJC1gv6JhWtAKPu3gfEZqZ5EXsHxDSUEOdxs9q9Dz74nuMA1eojkbL7oIscQFg5ZXwRUwnHzPyfb7nl+RrkNuqr3pDuK9X0gGi0sjBUNZlwbj7FasC2fP8zWXvHARRLI5yL2LT3ZngO/Fe1df81K+Y3289C9DLDWIPIxUVoD2SN3YTy1NUBZ0Jyfcpn9j6IZe/GHUKIsfQm4E8mO+EQYsT72D04zIW/njK6OyJ6Wxn2LiCTdZTC67HoTbgtAIworuPp54nqW7lwRR+mb0PCrdT9m2za8yD+rd2kpUMMMMxL56WE28qk+xZz395LifRdIFdjmVEqK86TpKUt7H5FSlIwtdmZqjo/sHWLLcJriMbkthhMMHVTkyh32bppvq1gPqKFimJKsX+zPwXIZggU74RZPjdJkthrX7u5TMziwnsMnqdw5fbrdkkjV/5D6BnNvPG5gD7ctpzB0A03fOIPGo3yAo3i2y2tNyWaXDV3U3fpQ9wQz+v3FZKPoIiqmttXAvLhavX7w5XKwl6bUUL/yUA+v5+YX4rDxS5mZm0vnPwFpLl0MEntzf/Ns0tCrJ6lzxD8w4svGHzm8IkXFnQebXbocGtYCKndfvvu9IknBv7kpZPyStHwW+T1N1NBiqfBcJMyeWFammuku+dZPSGU1PG9Da+//xtfP76nybSq1W122WVLDp/Xlz4jGq5xyyLaXroI6iIHVdnfnDOAN1yVnPhadeGOoGFDXui3FWCV2yzZL954uv2Y00I+x0paLxNKt1OK3zTrl3CWlUkb/eBQikcYe+kJDi87cdqLcIlvJ02PoNFg7qxhPZv2DY4vP49ofhvI5YSwGWSYWqNOiCKM+USlBZRKg2SNATzLmWpcTmmMfYGGf5yja0+waM9yovJrEF+KyFuJz9uAZ8fRxnFG/BiM1ElLfYQwSFxaSv1kwWR7FPchxkY/xNE1+5vnNlHgG1dX2yeu2e7MhcolTOCkZz7q4qPuPiomNXcZFfOamNda2/Lf3bzmxfb8t3w/cR91l9FsxjjITvTNHqVSvdexQciZFS4mxSdPe5O0CKlINcRDDat/eNEFA/8lL4TQujGvuebEIZEjv25p/ZOi4VirTmOzVqNT2NVM0BTHVCOTEB9yz/6vQPquavU9z7Q7AYq0RcPF2p+pjkGzraMoDMtN+ovtgbT15kvHf5dgrRTCTjjJeICqF7RIUQl4Fo9DVupRkFS1NKIarIitMRFJBTWcPG3O1fJ2HjKjoZRq6DnmWf2PLbLbtq8/+vBFF+1uuw/yfvL9i3Oc1eOpNK9JM60xyyIFuPLK4yPnzcs+hGXvFaI9QeNiPClSIL2Nkef0qqppKJ2wrLElqzdu+Ub1xR2txcEAEnvqqedruD2hWjohzb5a18c8G9sD9XEJrOn1D/A1MwMN7fsX9gd/cmysMTQ5rXLWEPL7BAHL+qifXEy9NrtPkzlqgLQxhPmjpx2ek7hy56uOoeEhQpQ7Yks9g3h6I9Rb9ImmqPQTQoWo52ZKpbcQ4lsJ0QbMLqZRGwSUuHcUZD+1l95Pze7k6CtypqZaJkQpUZybIhq1ftJ0JSJXEKI3EUpvRsONWHYJjbEBRCGeN4LZwzTGfpGjax5vJ7tDPcjJjHBm8axu5BWfFdP8T4H266gdtnVoN3OwZ7JBdqLvtKSvKBL0sKiWTaQPtzJ54QkDqSMyjPsQlu0Usb94tPrbDwM8MMkWXTwQtUrl/g+kfvKL6nabhJ5LgWW49UlegFVB6yI6jNgRS9OnTep/dnxo0WO33747bYZqnH9+ZN//QXZYNX7aMFQL35UEGo2TB0qlUsfsjgaMlDXeIRN0VDFERyRNR4AR1Z4draI2CrghOuI6Ntxxek6GNJSj/aj0mQYTXB1MpaSucqjt3Dvi8eoLB6+5ZvBOVasgvFajaK0QBtyZD152L7SWfC2WuiDH3bMhz+o7UR5UOfbQhmuxR5PEEhK9+sYoVQ0HBN1pmk2gJ5NakW43MaQqSUA0OhZC/DRCLG03mkjpsPjJ0eYSq0mSjFSrfLbuCx8LJreFKGxwD0vzXG0rjpVUJIwAx9zGnvEs+++qjYe2P/q+E52X+YVqlR0i4fEQlZY1tzuYalxv1EYeqX69FarTCpy/d6e7PR6intjVinPNXyBpdvJrPT3DwzOVmpsWlg0T9T4DVj4jI5ijBUNTRr/3GPN69p7u2i7jCPwVIaxFepSe82Cs9mpMHqdU3oPQh3kZiPHm85NnF0GooTJKo3GcNN2PNZ5ArMp7Xr13Qmrh86v3snTPHWR6IyLXEc9bBT6AWR9mEZiimiLRKBKOU39pH7XRv0PCF3jPq4YmO67yJ+uze2+g1LuZdGw5WTadwp3r6I3aX/Kq//W2ZFvFkkTs4986uQLxN6vPQV5b4eixzKvvW3teHmN1775V9ER/i9uaYvW0Dge6EfVAlj3N83922UwXr1K5v5yFk6s9s+UqMmDIAnWPwVLxMOyeHVHVg8C+SuXo6GzVmZtu+uT8kZFohUS+SmCxYX3iquJ+3NWPqLf6hElMJkn0tV/tX1YqlQbaOWFQVxdGouzY/k6LTV150yfnxyO6KgstVScGsiAWsrGDJ08Gi+Ppf69W33dicp+33bYlfv740Apx+jJrHRfU1cZKx77xjTtPmQPcZBqVyr19WQjLQ9YYNNEBy7yfQF4d3RkVYVjdh0APQe+havWOGsWSuW3ZNhEsXJGpz59MTzAZrlbv2teJhqtv3DQY123p1DeLpmPn6/6nvnjnuFzelOB27VobHTl+fJVYusKdpYL3g0YOI2I+BHJo3ryePQ8++JvHTzUHt922JT569IWVmUpvO90A3jN28B8e/A8d+kj06spPrw1ZiJvX7FTXa1b4410D1MMymqnFTWGoUXzP1G7/PxJljCF+75WHzogOgHt39SHzVhIKPpPKML3hEA1bTqO+gCjqwzxGPcI9ArW8iogWoTc+hDeGOLo2v36d1PymY2fZoX7Sl1biuhjxAdA+3CPUR3E5TqZH0Jf28Z6fG5qO3JzbbNqzgZ6+zaS1FTmX7Yj8DdKo/w090duS766oJ4nYJ58bXeaZ3+yEGMfOyktjBqpIJtX3ru3J04U2P7sGjf8WfNW0DNLdKPWAZzt41yt+YeoOE9G+/nG+ZOtLOjT0Xbv9dtL2dZFP19bTYgxJBBcW8/jdZimufK3safucSXWa/phKBW0vedUsk9XcNt3veYzf6fU78zEdeimqgrevTz15/NYa3zP1e/r05BELE49p+3WasI8Wc06SRHftIjp69EJtv4ZF37Ocg6nX9NTzOPGY2V2vU5Exi3VgZoWqwjY7Y+lxCj3NcJxpajlOe9wM+0zYv2CUrf4Vqkwc8+4ZUxJzbrP52Wso9W6mMbYan4FBaqRY+ijiv8Tzq4+TiG1+1hec9Nobxa0X1bP0oBpmmhJk+/f//P88kCSJsenZKwjRF4EFZOn0EmRpHmTpdt698vrZj9fK8ICm6jIXC4ZN7vfHbRGyHxXaM2pgbub63GFittWPN61dzAKniovsACFxZelzl1Cat5n62OXj3qGOfhkB1b1kY7/MC6/eTSJ27y7vS8NL17iEQU5Zx/HUUPfR1OZVhx/gRJKIsXnv2xG9H/N4gkNmAn1uxL2QNv6ad6+8bVYBsF100UUXp0CzWMUwaTact8fTuXJMKExrRqmnHymtgbtJ3PXoEDVTjoh7TfC647Uz/Yh4aipDw0O0ORDCL6AhHndZji9X10afA5aBUtjHZrn+bhdddNHFDMgZZNw4QTZ2pChZNFHymqzSZul84Cou/PU4AZLrJY0bHBHXE47XBK1LpnWh7XPKttcFr5tRH3Pbz7a7cxru/04ZYUPhYe6cqSPFtiyFzJ6d+ynqoosu/rUiZ5CH1p7A2UUUj+YS2jRhMyJKlsbEPeupp2uboVBHh847JioH1b2mntZUqam3fU7ZDjXB63h04OSreo/AxrwOx8n6G9FwMWld8WncP05RXUSOIeSOnblcg7aLLrr4V4vWUonC0+CdY+Pa4Q5ZuhbRm1m4u5ck0eR6SV+M4wOWlo5khLq518y9ZqH4tP/f3m7bniHHYi/tTUQsgTzfslS6sxhzyuJTEyGgYTcuh7r2xy666GKu0JLKgj5NOnaIEGkH70wbXHEvA/8WDVfkbnTX5OVSmzcW71NPjyleV3wio/S2Txtz1NTrkqbH5WR939G1jJK4suSpMpK9EwmvIa3TvnznFIgYuGHZDsbsBFw3RyENXXTRxb92FG5vMf7XoSNktpWoB5gpk4XcIQIr///27ifEruoO4Pj3d869972ZvsQYnTCRYEIYUpmFRBoGXdVAd13ZVpe1QWiKWVYLUkrvUIrYLooUq6YuFARtCy5aKaWbDLRKrS66KLY0dkwlZpKZMB3j+ObNfef+jov73sub/2/GSSPl94FhOMx973Bn8eOce3/n98P5H7L/vapgZR7d6RPS/O++xrRGuaROm1LGIJIUErQQ6fsJWlR/06IUuVxvNqY/Or7vWt7dGWvjXlz2CGW7AVvkcImAS66i5RvMjy2Sn7zpLWONMf8fVi4Vf/HPu3H+LYQM7ZSFiquu7tWHFCWtKaF4lVA8ztzs1W4CZh6jOzhDPSx/spdm0mg5XHSFYxnqaaaFoknQlk+GFubGaeYiSn4ugfuVQ++fILpniXo3ZTtZVeVj1ePRCN4r4v9AaJ3hyl0fbPsAvTHGbGDtXvr5f7+C9w91muC4zXfbUcnqBWX7t8TiKW6Nf+fd8dAfpPJzMeEIyUhzLoER5marPtj5SQnXM+MnYeTBYZyfIKs/g8a7KNsbTLpq/trwAq3mE8wee2GrrHhjjNmO6+Gv+3Lj7L++giQvEXWUUjcPkFW2tuLTgJbvoPpL2vIa82OLOZOdjhAb5CT2H/85cP5OvDyE84+AHKVsb/0cMaIkCSBTEB7mw7FLtno0xuymleEvzx2HH95LO/wY5Nuods4vbkkRgbQ2S2vpjzh+Ra35JqfuWVj3HGg3kD3z/ii++Bo++zqRE8Sy0TvJM8iczjtUH+Ty2GsrvtcYY3bB2kiUR8fBfxwn3fNzQjGBbljdp09nJQmQZAqySFieBvkLTt6mHS+RyiKxdJRxP94fBb5EZILa0CHay/XqxU/cOjjG7vPPuqLlr/mweQpWbuuNMWY3rB8gc1GeO/8NstrPCMVoFSQHLNsdY7Wa9KnDewgBNFR9dKvVaB2fgnMQ2lAG3TSNZ+0EikuA+FdieYqZV3Zem84YYzax/vY3jw75wu9pffIsiEOcDlyUVsQRoyMUyvKSom065wHrIBkxQnsZlpd08ODYPd0TOw165AKqP2UmTG/jXo0xZls2Xhbm0XHLhb0Mhadx8k1Uldh5ntjrM9qp5r3huG+K6+lBdBqUDPD5vjFU5eLTbJ6y/AHt1svMjTdta22MuVE2Xr3lonx05Bqe76O8iEsCzmkv6PWauMsm41U5jL1CE4N+vvsVUq0c01qL0H6C1L3I3G8sOBpjbqitHyzm0THy7gF88jhJ7Vto2IeuetPcW+XJjRgr3iuRi8T4JKfHzu74bo0xZhu2fv6XizI3PovwJGUxSZJdxGdVWbQYtfNWmV7zrN0aRxSRquct7k20/C4Mv3xD/xvGGNNnsLfHuSgzx+bJ0rOE9hkiUyRZwCeuU0OyIn1b452Pq+CbZHRSh14gLJ1hf/t1Zg62dnSXxhizA37gK6cmI/fcqnz8wHka8+dQvQJ6lNrQHlQFYlldGGVNy4beKrFroz7bUqXwJGmLMryDxu8RWs8xO36JuRG1Z47GmP+lwQMkwNRU5H4RFh+4xmO3vcFXH/0dZXsJn9ZIa/Wqx7QH5yIinf1ylPWDo4A4xbkqenrfojZ0haL1JzT8BIk/4jvH3mbiQCA/qUxNbqf5tTHGfGYDZn+vo9eshxRnXwAAALtJREFU+8uOO0aPojIBch/p8HGkPEQobyfGYbzXNdNEdagqIk18chHVC4Tib0TewvNnTn/xam8OSwI3xtwkOw+QcD2Adc9b73+vQcYhXLyDUu9E/GHSZBTxDaJmAGhs4uICoZyB+AGlTEOcxV+7zMzrrV4fW2OMuck+W4Bcrb8Rd34u4fCRhI9Dxp7EsdC5xgfFF8rwcOA/RwK5hF4tSAuMxpjPkd0NkP16W3BYWfJssjPu/LagaIz5nPoUBSp4D1AF9yMAAAAASUVORK5CYII=)"]},{"cell_type":"markdown","metadata":{"id":"UWTEBDfP4zHC"},"source":["[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/langtest/blob/main/demo/tutorials/llm_notebooks/dataset-notebooks/MultiLexSum_dataset.ipynb)"]},{"cell_type":"markdown","metadata":{"id":"wCxsD2KDAWU2"},"source":["**LangTest** is an open-source python library designed to help developers deliver safe and effective Natural Language Processing (NLP) models. Whether you are using **John Snow Labs, Hugging Face, Spacy** models or **OpenAI, Cohere, AI21, Hugging Face Inference API and Azure-OpenAI** based LLMs, it has got you covered. You can test any Named Entity Recognition (NER), Text Classification, fill-mask, Translation model using the library. We also support testing LLMS for Question-Answering, Summarization and text-generation tasks on benchmark datasets. The library supports 60+ out of the box tests. For a complete list of supported test categories, please refer to the [documentation](http://langtest.org/docs/pages/docs/test_categories).\n","\n","Metrics are calculated by comparing the model's extractions in the original list of sentences against the extractions carried out in the noisy list of sentences. The original annotated labels are not used at any point, we are simply comparing the model against itself in a 2 settings."]},{"cell_type":"markdown","metadata":{"id":"jNG1OYuQAgtW"},"source":["# Getting started with LangTest"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Y-cN_Woi4zHG"},"outputs":[],"source":["!pip install \"langtest[openai,transformers,evaluate]\""]},{"cell_type":"markdown","metadata":{"id":"EsEtlSiNAnSO"},"source":["# Harness and Its Parameters\n","\n","The Harness class is a testing class for Natural Language Processing (NLP) models. It evaluates the performance of a NLP model on a given task using test data and generates a report with test results.Harness can be imported from the LangTest library in the following way."]},{"cell_type":"code","execution_count":3,"metadata":{"id":"w2GPpdowS1C9"},"outputs":[],"source":["#Import Harness from the LangTest library\n","from langtest import Harness"]},{"cell_type":"markdown","metadata":{"id":"7_6PF_HGA4EO"},"source":["It imports the Harness class from within the module, that is designed to provide a blueprint or framework for conducting NLP testing, and that instances of the Harness class can be customized or configured for different testing scenarios or environments.\n","\n","Here is a list of the different parameters that can be passed to the Harness function:\n","\n","<br/>\n","\n","\n","| Parameter  | Description |  \n","| - | - | \n","|**task**     |Task for which the model is to be evaluated (question-answering or summarization)|\n","| **model**     | Specifies the model(s) to be evaluated. This parameter can be provided as either a dictionary or a list of dictionaries. Each dictionary should contain the following keys: <ul><li>model (mandatory): \tPipelineModel or path to a saved model or pretrained pipeline/model from hub.</li><li>hub (mandatory): Hub (library) to use in back-end for loading model from public models hub or from path</li></ul>|\n","| **data**      | The data to be used for evaluation. A dictionary providing flexibility and options for data sources. It should include the following keys: <ul><li>data_source (mandatory): The source of the data.</li><li>subset (optional): The subset of the data.</li><li>feature_column (optional): The column containing the features.</li><li>target_column (optional): The column containing the target labels.</li><li>split (optional): The data split to be used.</li><li>source (optional): Set to 'huggingface' when loading Hugging Face dataset.</li></ul> |\n","| **config**    | Configuration for the tests to be performed, specified in the form of a YAML file. |\n","\n","<br/>\n","<br/>"]},{"cell_type":"markdown","metadata":{"id":"pHJQHDcSA_CV"},"source":["# OpenAI Model Testing For Summarization\n","\n","In this section, we dive into testing of OpenAI models in summarization task.\n","\n","LangTest supports robustness tests for LLM testing for now."]},{"cell_type":"code","execution_count":3,"metadata":{"id":"YXVcv79JTAWA"},"outputs":[],"source":["import os\n","\n","os.environ[\"OPENAI_API_KEY\"] = \"<YOUR_API_KEY>\""]},{"cell_type":"markdown","metadata":{"id":"2Q1uClT2kgLB"},"source":["## MultiLexSum\n","[Multi-LexSum: Real-World Summaries of Civil Rights Lawsuits at Multiple Granularities](https://arxiv.org/abs/2206.10883)\n","\n","**Dataset Summary**\n","\n","The Multi-LexSum dataset consists of legal case summaries. The aim is for the model to thoroughly examine the given context and, upon understanding its content, produce a concise summary that captures the essential themes and key details.\n","\n","**Data Splits**\n","\n","- `test` :\tTesting set from the MultiLexSum dataset, containing 868 document and summary examples.\n","- `test-tiny` : Truncated version of XSum dataset which contains 50 document and summary examples."]},{"cell_type":"markdown","metadata":{"id":"1WO54aEnBKK8"},"source":["### Setup and Configure Harness"]},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":11,"status":"ok","timestamp":1692349537186,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"f13UydObTDRG","outputId":"b775e74b-3d8c-46e5-99b9-659a88ab3f48"},"outputs":[{"name":"stdout","output_type":"stream","text":["Test Configuration : \n"," {\n"," \"model_parameters\": {\n","  \"temperature\": 0.2,\n","  \"max_tokens\": 64\n"," },\n"," \"tests\": {\n","  \"defaults\": {\n","   \"min_pass_rate\": 1.0\n","  },\n","  \"robustness\": {\n","   \"add_typo\": {\n","    \"min_pass_rate\": 0.7\n","   },\n","   \"lowercase\": {\n","    \"min_pass_rate\": 0.7\n","   }\n","  }\n"," }\n","}\n"]}],"source":["harness = Harness(\n","                  task=\"summarization\", \n","                  model={\"model\": \"gpt-3.5-turbo-instruct\",\"hub\":\"openai\"}, \n","                  data={\"data_source\" :\"MultiLexSum\",\n","                        \"split\":\"test-tiny\"}\n","                  )"]},{"cell_type":"markdown","metadata":{"id":"djMJVtS3U3Wv"},"source":["## Robustness"]},{"cell_type":"markdown","metadata":{"id":"NQ1KF731BW5O"},"source":["For tests we used uppercase, Dyslexia Word Swap. Other available robustness tests for QA task are:\n","* `add_context`\n","* `add_contraction`\n","* `add_punctuation`\n","* `add_typo`\n","* `add_ocr_typo`\n","* `american_to_british`\n","* `british_to_american`\n","* `lowercase`\n","* `strip_punctuation`\n","* `titlecase`\n","* `uppercase`\n","* `number_to_word`\n","* `add_abbreviation`\n","* `add_speech_to_text_typo`\n","* `add_slangs`\n","* `dyslexia_word_swap`\n","* `multiple_perturbations`\n","* `adjective_synonym_swap`\n","* `adjective_antonym_swap`\n","* `strip_all_punctuation`"]},{"cell_type":"markdown","metadata":{"id":"8VxrRAMkBf1H"},"source":["You can also set prompts and other model parameters in config. Possible parameters are:\n","* `user_promt:` Promt to be given to the model.\n","* `temperature:` Temperature of the model.\n","* `max_tokens:` Maximum number of output tokens allowed for model."]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":10,"status":"ok","timestamp":1692349541501,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"fMFVq3mCTQ7j","outputId":"56588d33-a9c5-40ab-c05e-c4b836331c56"},"outputs":[{"data":{"text/plain":["{'evaluation': {'threshold': 0.5},\n"," 'tests': {'defaults': {'min_pass_rate': 0.65, 'threshold': 0.5},\n","  'robustness': {'uppercase': {'min_pass_rate': 0.66},\n","   'lowercase': {'min_pass_rate': 0.6}}}}"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["harness.configure(\n","{\n","\"evaluation\":{\"threshold\": 0.5},\n","\n"," 'tests': {'defaults': {'min_pass_rate': 0.65,\n","                        \"threshold\":0.50\n","                        },\n","           'robustness': {'uppercase': {'min_pass_rate': 0.66},\n","                          'lowercase':{'min_pass_rate': 0.60},\n","\n","                        }\n","          }\n"," }\n"," )"]},{"cell_type":"markdown","metadata":{"id":"lUDGc0nv4zHZ"},"source":["➤ The default metric for summarization is `rouge`. The other available metric is `bertscore` which can be initialised using -> `\"evaluation\":{\"metric\":\"bertscore\", \"threshold\": 0.5}`\n","\n","➤The default threshold value is `0.50`. If the eval_score is higher than threshold, then the \"pass\" will be as true.\n","\n","➤ You can adjust the level of transformation in the sentence by using the \"`prob`\" parameter, which controls the proportion of words to be changed during robustness tests.\n","\n","➤ **NOTE** : \"`prob`\" defaults to 1.0, which means all words will be transformed.\n","```\n","harness.configure(\n","{\n"," 'tests': {\n","    'defaults': {'min_pass_rate': 0.65},\n","      'robustness': {\n","        'uppercase': {'min_pass_rate': 0.66, 'prob': 0.50},\n","        'lowercase':{'min_pass_rate': 0.60, 'prob': 0.70},\n","      }\n","  }\n","})\n","\n","```"]},{"cell_type":"markdown","metadata":{"id":"m5IuCmiEBuW8"},"source":["Here we have configured the harness to perform robustness tests and defined the minimum pass rate for each test."]},{"cell_type":"code","execution_count":6,"metadata":{"id":"nmHqJ_TlUg8h"},"outputs":[],"source":["harness.data = harness.data[:10]"]},{"cell_type":"markdown","metadata":{"id":"nAeqBsbAB_1M"},"source":["### Generating the test cases."]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":13,"status":"ok","timestamp":1692349545289,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"CCJxFd4nUkMN","outputId":"5735c5fe-d31e-4736-f038-0b1f51e7e75c"},"outputs":[{"name":"stderr","output_type":"stream","text":["Generating testcases...: 100%|██████████| 1/1 [00:00<?, ?it/s]"]},{"name":"stderr","output_type":"stream","text":["\n"]},{"data":{"text/plain":[]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["harness.generate()"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":363},"executionInfo":{"elapsed":14,"status":"ok","timestamp":1692349546285,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"GVriwjmeo-H_","outputId":"e18e98cb-1aba-4057-b6cb-656022c3c1f6"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>original</th>\n","      <th>test_case</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On March 8th, 2014, several citizens of Montgo...</td>\n","      <td>ON MARCH 8TH, 2014, SEVERAL CITIZENS OF MONTGO...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On August 28, 2013, an indigent detainee in th...</td>\n","      <td>ON AUGUST 28, 2013, AN INDIGENT DETAINEE IN TH...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On May 1, 2006, an inmate awaiting execution a...</td>\n","      <td>ON MAY 1, 2006, AN INMATE AWAITING EXECUTION A...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On August 23, 2018, three Maricopa County, Ari...</td>\n","      <td>ON AUGUST 23, 2018, THREE MARICOPA COUNTY, ARI...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On March 8, 2006, the Pacific News Service fil...</td>\n","      <td>ON MARCH 8, 2006, THE PACIFIC NEWS SERVICE FIL...</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On April 20, 2012, a state prisoner filed this...</td>\n","      <td>ON APRIL 20, 2012, A STATE PRISONER FILED THIS...</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On June 9, 2018, the plaintiff in this case wa...</td>\n","      <td>ON JUNE 9, 2018, THE PLAINTIFF IN THIS CASE WA...</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On May 1, 2012, a D.C. resident whose car was ...</td>\n","      <td>ON MAY 1, 2012, A D.C. RESIDENT WHOSE CAR WAS ...</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>The city of Doraville relied on its municipal ...</td>\n","      <td>THE CITY OF DORAVILLE RELIED ON ITS MUNICIPAL ...</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On May 22, 2012, several national and local ne...</td>\n","      <td>ON MAY 22, 2012, SEVERAL NATIONAL AND LOCAL NE...</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On March 8th, 2014, several citizens of Montgo...</td>\n","      <td>on march 8th, 2014, several citizens of montgo...</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On August 28, 2013, an indigent detainee in th...</td>\n","      <td>on august 28, 2013, an indigent detainee in th...</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On May 1, 2006, an inmate awaiting execution a...</td>\n","      <td>on may 1, 2006, an inmate awaiting execution a...</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On August 23, 2018, three Maricopa County, Ari...</td>\n","      <td>on august 23, 2018, three maricopa county, ari...</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On March 8, 2006, the Pacific News Service fil...</td>\n","      <td>on march 8, 2006, the pacific news service fil...</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On April 20, 2012, a state prisoner filed this...</td>\n","      <td>on april 20, 2012, a state prisoner filed this...</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On June 9, 2018, the plaintiff in this case wa...</td>\n","      <td>on june 9, 2018, the plaintiff in this case wa...</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On May 1, 2012, a D.C. resident whose car was ...</td>\n","      <td>on may 1, 2012, a d.c. resident whose car was ...</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>The city of Doraville relied on its municipal ...</td>\n","      <td>the city of doraville relied on its municipal ...</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On May 22, 2012, several national and local ne...</td>\n","      <td>on may 22, 2012, several national and local ne...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["      category  test_type                                           original  \\\n","0   robustness  uppercase  On March 8th, 2014, several citizens of Montgo...   \n","1   robustness  uppercase  On August 28, 2013, an indigent detainee in th...   \n","2   robustness  uppercase  On May 1, 2006, an inmate awaiting execution a...   \n","3   robustness  uppercase  On August 23, 2018, three Maricopa County, Ari...   \n","4   robustness  uppercase  On March 8, 2006, the Pacific News Service fil...   \n","5   robustness  uppercase  On April 20, 2012, a state prisoner filed this...   \n","6   robustness  uppercase  On June 9, 2018, the plaintiff in this case wa...   \n","7   robustness  uppercase  On May 1, 2012, a D.C. resident whose car was ...   \n","8   robustness  uppercase  The city of Doraville relied on its municipal ...   \n","9   robustness  uppercase  On May 22, 2012, several national and local ne...   \n","10  robustness  lowercase  On March 8th, 2014, several citizens of Montgo...   \n","11  robustness  lowercase  On August 28, 2013, an indigent detainee in th...   \n","12  robustness  lowercase  On May 1, 2006, an inmate awaiting execution a...   \n","13  robustness  lowercase  On August 23, 2018, three Maricopa County, Ari...   \n","14  robustness  lowercase  On March 8, 2006, the Pacific News Service fil...   \n","15  robustness  lowercase  On April 20, 2012, a state prisoner filed this...   \n","16  robustness  lowercase  On June 9, 2018, the plaintiff in this case wa...   \n","17  robustness  lowercase  On May 1, 2012, a D.C. resident whose car was ...   \n","18  robustness  lowercase  The city of Doraville relied on its municipal ...   \n","19  robustness  lowercase  On May 22, 2012, several national and local ne...   \n","\n","                                            test_case  \n","0   ON MARCH 8TH, 2014, SEVERAL CITIZENS OF MONTGO...  \n","1   ON AUGUST 28, 2013, AN INDIGENT DETAINEE IN TH...  \n","2   ON MAY 1, 2006, AN INMATE AWAITING EXECUTION A...  \n","3   ON AUGUST 23, 2018, THREE MARICOPA COUNTY, ARI...  \n","4   ON MARCH 8, 2006, THE PACIFIC NEWS SERVICE FIL...  \n","5   ON APRIL 20, 2012, A STATE PRISONER FILED THIS...  \n","6   ON JUNE 9, 2018, THE PLAINTIFF IN THIS CASE WA...  \n","7   ON MAY 1, 2012, A D.C. RESIDENT WHOSE CAR WAS ...  \n","8   THE CITY OF DORAVILLE RELIED ON ITS MUNICIPAL ...  \n","9   ON MAY 22, 2012, SEVERAL NATIONAL AND LOCAL NE...  \n","10  on march 8th, 2014, several citizens of montgo...  \n","11  on august 28, 2013, an indigent detainee in th...  \n","12  on may 1, 2006, an inmate awaiting execution a...  \n","13  on august 23, 2018, three maricopa county, ari...  \n","14  on march 8, 2006, the pacific news service fil...  \n","15  on april 20, 2012, a state prisoner filed this...  \n","16  on june 9, 2018, the plaintiff in this case wa...  \n","17  on may 1, 2012, a d.c. resident whose car was ...  \n","18  the city of doraville relied on its municipal ...  \n","19  on may 22, 2012, several national and local ne...  "]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["harness.testcases()"]},{"cell_type":"markdown","metadata":{"id":"ZEWchFb8CDrk"},"source":["harness.generate() method automatically generates the test cases (based on the provided configuration)"]},{"cell_type":"markdown","metadata":{"id":"MEnLcl-OCG1O"},"source":["### Running the tests"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":36091,"status":"ok","timestamp":1692349583122,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"gFEez-T0UlcC","outputId":"cdb22cdf-259b-49a7-85e0-ae510909d5bb"},"outputs":[{"name":"stderr","output_type":"stream","text":["Running testcases... : 100%|██████████| 20/20 [01:27<00:00,  4.37s/it]\n"]},{"data":{"text/plain":[]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["harness.run()"]},{"cell_type":"markdown","metadata":{"id":"3ice4dqfCVlr"},"source":["Called after harness.generate() and is to used to run all the tests.  Returns a pass/fail flag for each test."]},{"cell_type":"markdown","metadata":{"id":"g1NxuqveOc-t"},"source":["### Generated Results"]},{"cell_type":"code","execution_count":12,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":568,"referenced_widgets":["ddda15243d9045eea1b65e0ab6b07d6a","bbca32416af74cd0be3c5615e299fb2f","ebf8dd327f784508888ea4687e0bdb5a","53406674f9604befbddb06a33c85561e","356179558554416c84cf0b16bd2eedf2","2e5772c24a404bcaab382dd09a3498d0","aa4207cfcbac44929d9841eabbd8954b","fc16bc00006b43adb9d43ab2c4621c51","f49335df030645e4b2ce5c3fffa689bd","8d70d582cd6f43f596bfb1590c215164","5f6752be51ef474d850047a110135f14"]},"executionInfo":{"elapsed":23434,"status":"ok","timestamp":1692349671039,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"ZjYBONiuYJdK","outputId":"2029d9e8-9d21-443d-f10e-1ae1237a8dfc"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>original</th>\n","      <th>test_case</th>\n","      <th>expected_result</th>\n","      <th>actual_result</th>\n","      <th>eval_score</th>\n","      <th>pass</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On March 8th, 2014, several citizens of Montgo...</td>\n","      <td>ON MARCH 8TH, 2014, SEVERAL CITIZENS OF MONTGO...</td>\n","      <td>On March 8th, 2014, several citizens of Montg...</td>\n","      <td>\\nIn March 2014, several citizens of Montgomer...</td>\n","      <td>0.304762</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On August 28, 2013, an indigent detainee in th...</td>\n","      <td>ON AUGUST 28, 2013, AN INDIGENT DETAINEE IN TH...</td>\n","      <td>\\nIn August 2013, an indigent detainee in the ...</td>\n","      <td>On August 28, 2013, an indigent detainee in t...</td>\n","      <td>0.647619</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On May 1, 2006, an inmate awaiting execution a...</td>\n","      <td>ON MAY 1, 2006, AN INMATE AWAITING EXECUTION A...</td>\n","      <td>\\nIn 2006, two inmates in the Arkansas Departm...</td>\n","      <td>\\n\\nIn May 2006, an inmate awaiting execution ...</td>\n","      <td>0.594059</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On August 23, 2018, three Maricopa County, Ari...</td>\n","      <td>ON AUGUST 23, 2018, THREE MARICOPA COUNTY, ARI...</td>\n","      <td>\\nOn August 23, 2018, three Maricopa County, A...</td>\n","      <td>\\n\\nOn August 23, 2018, three Maricopa County,...</td>\n","      <td>0.903226</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On March 8, 2006, the Pacific News Service fil...</td>\n","      <td>ON MARCH 8, 2006, THE PACIFIC NEWS SERVICE FIL...</td>\n","      <td>On March 8, 2006, Pacific News Service filed ...</td>\n","      <td>\\n\\nOn March 8, 2006, Pacific News Service fil...</td>\n","      <td>0.547170</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On April 20, 2012, a state prisoner filed this...</td>\n","      <td>ON APRIL 20, 2012, A STATE PRISONER FILED THIS...</td>\n","      <td>\\nIn April 2012, a state prisoner filed a clas...</td>\n","      <td>\\n\\nIn April 2012, a state prisoner filed a cl...</td>\n","      <td>0.596154</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On June 9, 2018, the plaintiff in this case wa...</td>\n","      <td>ON JUNE 9, 2018, THE PLAINTIFF IN THIS CASE WA...</td>\n","      <td>\\n\\nIn June 2018, the plaintiff was arrested i...</td>\n","      <td>\\n\\nOn June 9, 2018, a plaintiff was arrested ...</td>\n","      <td>0.849057</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On May 1, 2012, a D.C. resident whose car was ...</td>\n","      <td>ON MAY 1, 2012, A D.C. RESIDENT WHOSE CAR WAS ...</td>\n","      <td>\\nIn May 2012, a D.C. resident whose car was s...</td>\n","      <td>\\n\\nOn May 1, 2012, a D.C. resident filed a la...</td>\n","      <td>0.653846</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>The city of Doraville relied on its municipal ...</td>\n","      <td>THE CITY OF DORAVILLE RELIED ON ITS MUNICIPAL ...</td>\n","      <td>\\nIn May 2018, four individuals filed a lawsui...</td>\n","      <td>\\nFour individuals filed a lawsuit against the...</td>\n","      <td>0.640777</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>On May 22, 2012, several national and local ne...</td>\n","      <td>ON MAY 22, 2012, SEVERAL NATIONAL AND LOCAL NE...</td>\n","      <td>On May 22, 2012, several news agencies filed ...</td>\n","      <td>\\n\\nIn May 2012, several news agencies filed a...</td>\n","      <td>0.601942</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On March 8th, 2014, several citizens of Montgo...</td>\n","      <td>on march 8th, 2014, several citizens of montgo...</td>\n","      <td>\\nIn March 2014, several citizens of Montgomer...</td>\n","      <td>\\nIn March 2014, several citizens of Montgomer...</td>\n","      <td>0.504854</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On August 28, 2013, an indigent detainee in th...</td>\n","      <td>on august 28, 2013, an indigent detainee in th...</td>\n","      <td>\\nTwo indigent detainees in the Montgomery Mun...</td>\n","      <td>\\n\\nIn August 2013, an indigent detainee in th...</td>\n","      <td>0.477064</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On May 1, 2006, an inmate awaiting execution a...</td>\n","      <td>on may 1, 2006, an inmate awaiting execution a...</td>\n","      <td>\\nIn 2006, two inmates in the Arkansas Departm...</td>\n","      <td>\\n\\nIn 2006, two inmates awaiting execution at...</td>\n","      <td>0.504505</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On August 23, 2018, three Maricopa County, Ari...</td>\n","      <td>on august 23, 2018, three maricopa county, ari...</td>\n","      <td>\\n\\nOn August 23, 2018, three Maricopa County,...</td>\n","      <td>\\n\\nOn August 23, 2018, three Maricopa County,...</td>\n","      <td>0.652174</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On March 8, 2006, the Pacific News Service fil...</td>\n","      <td>on march 8, 2006, the pacific news service fil...</td>\n","      <td>On March 8, 2006, the Pacific News Service fi...</td>\n","      <td>\\n\\nIn 2006, the Pacific News Service filed a ...</td>\n","      <td>0.764706</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On April 20, 2012, a state prisoner filed this...</td>\n","      <td>on april 20, 2012, a state prisoner filed this...</td>\n","      <td>\\nIn April 2012, a state prisoner filed a clas...</td>\n","      <td>In April 2012, a state prisoner filed a class...</td>\n","      <td>0.892857</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On June 9, 2018, the plaintiff in this case wa...</td>\n","      <td>on june 9, 2018, the plaintiff in this case wa...</td>\n","      <td>\\nThe plaintiff was arrested in Denver, Colora...</td>\n","      <td>\\n\\nThe plaintiff was arrested in Denver, Colo...</td>\n","      <td>0.880734</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On May 1, 2012, a D.C. resident whose car was ...</td>\n","      <td>on may 1, 2012, a d.c. resident whose car was ...</td>\n","      <td>On May 1, 2012, a D.C. resident filed a lawsu...</td>\n","      <td>\\n\\nOn May 1, 2012, a D.C. resident filed a la...</td>\n","      <td>0.826923</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>The city of Doraville relied on its municipal ...</td>\n","      <td>the city of doraville relied on its municipal ...</td>\n","      <td>\\nIn May 2018, four individuals filed a lawsui...</td>\n","      <td>\\nFour individuals filed a lawsuit against the...</td>\n","      <td>0.819048</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>On May 22, 2012, several national and local ne...</td>\n","      <td>on may 22, 2012, several national and local ne...</td>\n","      <td>On May 22, 2012, several news agencies filed ...</td>\n","      <td>\\n\\nOn May 22, 2012, news agencies filed a law...</td>\n","      <td>0.698113</td>\n","      <td>True</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["      category  test_type                                           original  \\\n","0   robustness  uppercase  On March 8th, 2014, several citizens of Montgo...   \n","1   robustness  uppercase  On August 28, 2013, an indigent detainee in th...   \n","2   robustness  uppercase  On May 1, 2006, an inmate awaiting execution a...   \n","3   robustness  uppercase  On August 23, 2018, three Maricopa County, Ari...   \n","4   robustness  uppercase  On March 8, 2006, the Pacific News Service fil...   \n","5   robustness  uppercase  On April 20, 2012, a state prisoner filed this...   \n","6   robustness  uppercase  On June 9, 2018, the plaintiff in this case wa...   \n","7   robustness  uppercase  On May 1, 2012, a D.C. resident whose car was ...   \n","8   robustness  uppercase  The city of Doraville relied on its municipal ...   \n","9   robustness  uppercase  On May 22, 2012, several national and local ne...   \n","10  robustness  lowercase  On March 8th, 2014, several citizens of Montgo...   \n","11  robustness  lowercase  On August 28, 2013, an indigent detainee in th...   \n","12  robustness  lowercase  On May 1, 2006, an inmate awaiting execution a...   \n","13  robustness  lowercase  On August 23, 2018, three Maricopa County, Ari...   \n","14  robustness  lowercase  On March 8, 2006, the Pacific News Service fil...   \n","15  robustness  lowercase  On April 20, 2012, a state prisoner filed this...   \n","16  robustness  lowercase  On June 9, 2018, the plaintiff in this case wa...   \n","17  robustness  lowercase  On May 1, 2012, a D.C. resident whose car was ...   \n","18  robustness  lowercase  The city of Doraville relied on its municipal ...   \n","19  robustness  lowercase  On May 22, 2012, several national and local ne...   \n","\n","                                            test_case  \\\n","0   ON MARCH 8TH, 2014, SEVERAL CITIZENS OF MONTGO...   \n","1   ON AUGUST 28, 2013, AN INDIGENT DETAINEE IN TH...   \n","2   ON MAY 1, 2006, AN INMATE AWAITING EXECUTION A...   \n","3   ON AUGUST 23, 2018, THREE MARICOPA COUNTY, ARI...   \n","4   ON MARCH 8, 2006, THE PACIFIC NEWS SERVICE FIL...   \n","5   ON APRIL 20, 2012, A STATE PRISONER FILED THIS...   \n","6   ON JUNE 9, 2018, THE PLAINTIFF IN THIS CASE WA...   \n","7   ON MAY 1, 2012, A D.C. RESIDENT WHOSE CAR WAS ...   \n","8   THE CITY OF DORAVILLE RELIED ON ITS MUNICIPAL ...   \n","9   ON MAY 22, 2012, SEVERAL NATIONAL AND LOCAL NE...   \n","10  on march 8th, 2014, several citizens of montgo...   \n","11  on august 28, 2013, an indigent detainee in th...   \n","12  on may 1, 2006, an inmate awaiting execution a...   \n","13  on august 23, 2018, three maricopa county, ari...   \n","14  on march 8, 2006, the pacific news service fil...   \n","15  on april 20, 2012, a state prisoner filed this...   \n","16  on june 9, 2018, the plaintiff in this case wa...   \n","17  on may 1, 2012, a d.c. resident whose car was ...   \n","18  the city of doraville relied on its municipal ...   \n","19  on may 22, 2012, several national and local ne...   \n","\n","                                      expected_result  \\\n","0    On March 8th, 2014, several citizens of Montg...   \n","1   \\nIn August 2013, an indigent detainee in the ...   \n","2   \\nIn 2006, two inmates in the Arkansas Departm...   \n","3   \\nOn August 23, 2018, three Maricopa County, A...   \n","4    On March 8, 2006, Pacific News Service filed ...   \n","5   \\nIn April 2012, a state prisoner filed a clas...   \n","6   \\n\\nIn June 2018, the plaintiff was arrested i...   \n","7   \\nIn May 2012, a D.C. resident whose car was s...   \n","8   \\nIn May 2018, four individuals filed a lawsui...   \n","9    On May 22, 2012, several news agencies filed ...   \n","10  \\nIn March 2014, several citizens of Montgomer...   \n","11  \\nTwo indigent detainees in the Montgomery Mun...   \n","12  \\nIn 2006, two inmates in the Arkansas Departm...   \n","13  \\n\\nOn August 23, 2018, three Maricopa County,...   \n","14   On March 8, 2006, the Pacific News Service fi...   \n","15  \\nIn April 2012, a state prisoner filed a clas...   \n","16  \\nThe plaintiff was arrested in Denver, Colora...   \n","17   On May 1, 2012, a D.C. resident filed a lawsu...   \n","18  \\nIn May 2018, four individuals filed a lawsui...   \n","19   On May 22, 2012, several news agencies filed ...   \n","\n","                                        actual_result  eval_score   pass  \n","0   \\nIn March 2014, several citizens of Montgomer...    0.304762  False  \n","1    On August 28, 2013, an indigent detainee in t...    0.647619   True  \n","2   \\n\\nIn May 2006, an inmate awaiting execution ...    0.594059   True  \n","3   \\n\\nOn August 23, 2018, three Maricopa County,...    0.903226   True  \n","4   \\n\\nOn March 8, 2006, Pacific News Service fil...    0.547170   True  \n","5   \\n\\nIn April 2012, a state prisoner filed a cl...    0.596154   True  \n","6   \\n\\nOn June 9, 2018, a plaintiff was arrested ...    0.849057   True  \n","7   \\n\\nOn May 1, 2012, a D.C. resident filed a la...    0.653846   True  \n","8   \\nFour individuals filed a lawsuit against the...    0.640777   True  \n","9   \\n\\nIn May 2012, several news agencies filed a...    0.601942   True  \n","10  \\nIn March 2014, several citizens of Montgomer...    0.504854   True  \n","11  \\n\\nIn August 2013, an indigent detainee in th...    0.477064  False  \n","12  \\n\\nIn 2006, two inmates awaiting execution at...    0.504505   True  \n","13  \\n\\nOn August 23, 2018, three Maricopa County,...    0.652174   True  \n","14  \\n\\nIn 2006, the Pacific News Service filed a ...    0.764706   True  \n","15   In April 2012, a state prisoner filed a class...    0.892857   True  \n","16  \\n\\nThe plaintiff was arrested in Denver, Colo...    0.880734   True  \n","17  \\n\\nOn May 1, 2012, a D.C. resident filed a la...    0.826923   True  \n","18  \\nFour individuals filed a lawsuit against the...    0.819048   True  \n","19  \\n\\nOn May 22, 2012, news agencies filed a law...    0.698113   True  "]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["harness.generated_results()"]},{"cell_type":"markdown","metadata":{"id":"Gl5QGV9pCZfz"},"source":["This method returns the generated results in the form of a pandas dataframe, which provides a convenient and easy-to-use format for working with the test results. You can use this method to quickly identify the test cases that failed and to determine where fixes are needed."]},{"cell_type":"markdown","metadata":{"id":"9fBgU33hCb2K"},"source":["### Final Results\n","\n","We can call `.report()` which summarizes the results giving information about pass and fail counts and overall test pass/fail flag."]},{"cell_type":"code","execution_count":13,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"executionInfo":{"elapsed":5571,"status":"ok","timestamp":1692349676596,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"nDmRw1AeUqIl","outputId":"77be0ba1-7dd6-48da-9bb0-8f507852d401"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>fail_count</th>\n","      <th>pass_count</th>\n","      <th>pass_rate</th>\n","      <th>minimum_pass_rate</th>\n","      <th>pass</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>robustness</td>\n","      <td>uppercase</td>\n","      <td>1</td>\n","      <td>9</td>\n","      <td>90%</td>\n","      <td>66%</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>robustness</td>\n","      <td>lowercase</td>\n","      <td>1</td>\n","      <td>9</td>\n","      <td>90%</td>\n","      <td>60%</td>\n","      <td>True</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["     category  test_type  fail_count  pass_count pass_rate minimum_pass_rate  \\\n","0  robustness  uppercase           1           9       90%               66%   \n","1  robustness  lowercase           1           9       90%               60%   \n","\n","   pass  \n","0  True  \n","1  True  "]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["harness.report()"]},{"cell_type":"markdown","metadata":{"id":"IULGQtWAWp4L"},"source":["## Fairness"]},{"cell_type":"markdown","metadata":{"id":"z85d594ZGXyX"},"source":["Available Fairness tests for QA task are:\n","\n","* `max_gender_rouge1_score`\n","* `max_gender_rouge2_score`\n","* `max_gender_rougeL_score`\n","* `max_gender_rougeLsum_score`\n","* `min_gender_rouge1_score`\n","* `min_gender_rouge2_score`\n","* `min_gender_rougeL_score`\n","* `min_gender_rougeLsum_score`"]},{"cell_type":"code","execution_count":15,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":21,"status":"ok","timestamp":1692349676598,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"OoMGAn_FWpaP","outputId":"c59d3efe-12e9-474d-aa18-253c3b37f68c"},"outputs":[{"name":"stdout","output_type":"stream","text":["Test Configuration : \n"," {\n"," \"model_parameters\": {\n","  \"temperature\": 0.2,\n","  \"max_tokens\": 64\n"," },\n"," \"tests\": {\n","  \"defaults\": {\n","   \"min_pass_rate\": 1.0\n","  },\n","  \"robustness\": {\n","   \"add_typo\": {\n","    \"min_pass_rate\": 0.7\n","   },\n","   \"lowercase\": {\n","    \"min_pass_rate\": 0.7\n","   }\n","  }\n"," }\n","}\n"]}],"source":["harness = Harness(\n","                  task=\"summarization\", \n","                  model={\"model\": \"gpt-3.5-turbo-instruct\",\"hub\":\"openai\"}, \n","                  data={\"data_source\" :\"MultiLexSum\",\n","                        \"split\":\"test-tiny\"}\n","                  )"]},{"cell_type":"code","execution_count":16,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":69,"status":"ok","timestamp":1692349677392,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"45-rhwhTXMWb","outputId":"ceb4f8ed-b6e1-4b73-b15a-76e85e54a71e"},"outputs":[{"data":{"text/plain":["{'tests': {'defaults': {'min_pass_rate': 0.65},\n","  'fairness': {'min_gender_rouge1_score': {'min_score': 0.66},\n","   'min_gender_rouge2_score': {'min_score': 0.6},\n","   'min_gender_rougeL_score': {'min_score': 0.66},\n","   'min_gender_rougeLsum_score': {'min_score': 0.66},\n","   'max_gender_rouge1_score': {'max_score': 0.66},\n","   'max_gender_rouge2_score': {'max_score': 0.6},\n","   'max_gender_rougeL_score': {'max_score': 0.66},\n","   'max_gender_rougeLsum_score': {'max_score': 0.66}}}}"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["harness.configure(\n","{\n"," 'tests': {'defaults': {'min_pass_rate': 0.65},\n","           'fairness': {\n","               'min_gender_rouge1_score': {'min_score': 0.66},\n","               'min_gender_rouge2_score':{'min_score': 0.60},\n","               'min_gender_rougeL_score': {'min_score': 0.66},\n","               'min_gender_rougeLsum_score': {'min_score': 0.66},\n","               'max_gender_rouge1_score': {'max_score': 0.66},\n","               'max_gender_rouge2_score':{'max_score': 0.60},\n","               'max_gender_rougeL_score': {'max_score': 0.66},\n","               'max_gender_rougeLsum_score': {'max_score': 0.66},\n","\n","\n","\n","\n","                        }\n","          }\n"," }\n"," )"]},{"cell_type":"code","execution_count":17,"metadata":{"id":"U8QFkedl4zHq"},"outputs":[],"source":["harness.data = harness.data[:10]"]},{"cell_type":"markdown","metadata":{"id":"dw85pgowGx8t"},"source":["### Generating the Test Cases"]},{"cell_type":"code","execution_count":18,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":65,"status":"ok","timestamp":1692349677395,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"F2p1pXfoXzND","outputId":"45a1f491-b8dc-4929-97d1-cbe07093daa5"},"outputs":[{"name":"stderr","output_type":"stream","text":["Generating testcases...: 100%|██████████| 1/1 [00:00<00:00, 662.29it/s]\n"]},{"data":{"text/plain":[]},"execution_count":18,"metadata":{},"output_type":"execute_result"}],"source":["harness.generate()"]},{"cell_type":"code","execution_count":19,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":802},"executionInfo":{"elapsed":54,"status":"ok","timestamp":1692349677396,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"vJZxMYyKX0Pe","outputId":"2a2eeb09-cc48-4b39-e0cf-a1cc25ca4688"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>test_case</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge1_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge1_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge1_score</td>\n","      <td>unknown</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge2_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge2_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge2_score</td>\n","      <td>unknown</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeL_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeL_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeL_score</td>\n","      <td>unknown</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeLsum_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeLsum_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeLsum_score</td>\n","      <td>unknown</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge1_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge1_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge1_score</td>\n","      <td>unknown</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge2_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge2_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge2_score</td>\n","      <td>unknown</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeL_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeL_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeL_score</td>\n","      <td>unknown</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeLsum_score</td>\n","      <td>male</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeLsum_score</td>\n","      <td>female</td>\n","    </tr>\n","    <tr>\n","      <th>23</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeLsum_score</td>\n","      <td>unknown</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["    category                   test_type test_case\n","0   fairness     min_gender_rouge1_score      male\n","1   fairness     min_gender_rouge1_score    female\n","2   fairness     min_gender_rouge1_score   unknown\n","3   fairness     min_gender_rouge2_score      male\n","4   fairness     min_gender_rouge2_score    female\n","5   fairness     min_gender_rouge2_score   unknown\n","6   fairness     min_gender_rougeL_score      male\n","7   fairness     min_gender_rougeL_score    female\n","8   fairness     min_gender_rougeL_score   unknown\n","9   fairness  min_gender_rougeLsum_score      male\n","10  fairness  min_gender_rougeLsum_score    female\n","11  fairness  min_gender_rougeLsum_score   unknown\n","12  fairness     max_gender_rouge1_score      male\n","13  fairness     max_gender_rouge1_score    female\n","14  fairness     max_gender_rouge1_score   unknown\n","15  fairness     max_gender_rouge2_score      male\n","16  fairness     max_gender_rouge2_score    female\n","17  fairness     max_gender_rouge2_score   unknown\n","18  fairness     max_gender_rougeL_score      male\n","19  fairness     max_gender_rougeL_score    female\n","20  fairness     max_gender_rougeL_score   unknown\n","21  fairness  max_gender_rougeLsum_score      male\n","22  fairness  max_gender_rougeLsum_score    female\n","23  fairness  max_gender_rougeLsum_score   unknown"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["harness.testcases()"]},{"cell_type":"markdown","metadata":{"id":"zSgEmwr7G2Xl"},"source":["### Running the tests"]},{"cell_type":"code","execution_count":20,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":149,"referenced_widgets":["c14c5775e4194149bb4cffce1bc980dd","56ac8962b6ca4aa7a3644739a5ccc611","33bc82cae06a436fa02cba33d7431810","c4e8c8cde5ac4ac5b7f3bb5e8e1dadcd","144e64d2603f4edda5d3493a7c8c2fb1","439ce4d6d29e467fa28ce4fbfd6926c4","fccc66893beb4f33b1667972f326f29d","190cd5e52934428abd68de51c6ec3212","2781c2444a8e4203b0083c97629fcf5f","84c69aafc65c4886ac0677f7c8a449d7","3ee2bf0fd98a451faeb9509fda44403f","a4a3b95dbd5746d69edd20f5f25bb203","59d57d203be3423c91c901da7f86aac5","9258191dffaf4e4e83d73eab458267a1","3990f2d5120843278eadbd9cbc21a056","99a4be421a2241bb8d9966eae7def4b0","d71dd704a9de42538a43992bbf608b87","968cd355c9b648cfa73d83f0578b5407","41af75b0a8b54e8782d68579ac379905","2546ce703ea0478da065d1698e955caf","bf662816272c441d9f0041fa9cf67e14","73bade4962954c758e7554dd742c5812","38bd875b2a9b4e3c908c60b438cdc00a","e78351f3743c46a683c40b77e39cec0a","b80ee92dce9a474295c223cd6ee7f7da","a91fb540bb044a51b85938a3f5dfac39","27c790022b4f482fae6a826aa7fe005c","8bbc85420fbd4715a361f95f0018e83d","0b18eaae9df349dc89d5b889d806bb00","9245e5d234bd430e81187fb4dae8fbde","762aefb0bdb34353955c1069067f0710","73b4108a58ec4de7bf1909715d5b04d3","edc1ea93d9ab4e4587a5bf491d495713"]},"executionInfo":{"elapsed":22902,"status":"ok","timestamp":1692349700247,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"marZgGMEX2F1","outputId":"83d580ad-1a07-428c-9030-2a2229491385"},"outputs":[{"name":"stderr","output_type":"stream","text":["Running testcases... :   0%|          | 0/24 [00:00<?, ?it/s]"]},{"name":"stderr","output_type":"stream","text":["Running testcases... : 100%|██████████| 24/24 [06:01<00:00, 15.04s/it]\n"]},{"data":{"text/plain":[]},"execution_count":20,"metadata":{},"output_type":"execute_result"}],"source":["harness.run()"]},{"cell_type":"markdown","metadata":{"id":"syaSCLsQIGiV"},"source":["### Generated Results"]},{"cell_type":"code","execution_count":22,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":802},"executionInfo":{"elapsed":173,"status":"ok","timestamp":1692349700251,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"ZoI8_JUBX4XC","outputId":"10cc58d4-7d31-467b-a9c8-55d0e62e0815"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>test_case</th>\n","      <th>expected_result</th>\n","      <th>actual_result</th>\n","      <th>pass</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge1_score</td>\n","      <td>male</td>\n","      <td>0.66</td>\n","      <td>0.431206</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge1_score</td>\n","      <td>female</td>\n","      <td>0.66</td>\n","      <td>0.322581</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge1_score</td>\n","      <td>unknown</td>\n","      <td>0.66</td>\n","      <td>0.389023</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge2_score</td>\n","      <td>male</td>\n","      <td>0.60</td>\n","      <td>0.248398</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge2_score</td>\n","      <td>female</td>\n","      <td>0.60</td>\n","      <td>0.086957</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge2_score</td>\n","      <td>unknown</td>\n","      <td>0.60</td>\n","      <td>0.253425</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeL_score</td>\n","      <td>male</td>\n","      <td>0.66</td>\n","      <td>0.355613</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeL_score</td>\n","      <td>female</td>\n","      <td>0.66</td>\n","      <td>0.172043</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeL_score</td>\n","      <td>unknown</td>\n","      <td>0.66</td>\n","      <td>0.326059</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeLsum_score</td>\n","      <td>male</td>\n","      <td>0.66</td>\n","      <td>0.357904</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeLsum_score</td>\n","      <td>female</td>\n","      <td>0.66</td>\n","      <td>0.172043</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeLsum_score</td>\n","      <td>unknown</td>\n","      <td>0.66</td>\n","      <td>0.326059</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge1_score</td>\n","      <td>male</td>\n","      <td>0.66</td>\n","      <td>0.431206</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge1_score</td>\n","      <td>female</td>\n","      <td>0.66</td>\n","      <td>0.322581</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge1_score</td>\n","      <td>unknown</td>\n","      <td>0.66</td>\n","      <td>0.389023</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge2_score</td>\n","      <td>male</td>\n","      <td>0.60</td>\n","      <td>0.248398</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge2_score</td>\n","      <td>female</td>\n","      <td>0.60</td>\n","      <td>0.086957</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge2_score</td>\n","      <td>unknown</td>\n","      <td>0.60</td>\n","      <td>0.253425</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeL_score</td>\n","      <td>male</td>\n","      <td>0.66</td>\n","      <td>0.355613</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeL_score</td>\n","      <td>female</td>\n","      <td>0.66</td>\n","      <td>0.172043</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeL_score</td>\n","      <td>unknown</td>\n","      <td>0.66</td>\n","      <td>0.326059</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeLsum_score</td>\n","      <td>male</td>\n","      <td>0.66</td>\n","      <td>0.357904</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeLsum_score</td>\n","      <td>female</td>\n","      <td>0.66</td>\n","      <td>0.172043</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>23</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeLsum_score</td>\n","      <td>unknown</td>\n","      <td>0.66</td>\n","      <td>0.326059</td>\n","      <td>True</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["    category                   test_type test_case  expected_result  \\\n","0   fairness     min_gender_rouge1_score      male             0.66   \n","1   fairness     min_gender_rouge1_score    female             0.66   \n","2   fairness     min_gender_rouge1_score   unknown             0.66   \n","3   fairness     min_gender_rouge2_score      male             0.60   \n","4   fairness     min_gender_rouge2_score    female             0.60   \n","5   fairness     min_gender_rouge2_score   unknown             0.60   \n","6   fairness     min_gender_rougeL_score      male             0.66   \n","7   fairness     min_gender_rougeL_score    female             0.66   \n","8   fairness     min_gender_rougeL_score   unknown             0.66   \n","9   fairness  min_gender_rougeLsum_score      male             0.66   \n","10  fairness  min_gender_rougeLsum_score    female             0.66   \n","11  fairness  min_gender_rougeLsum_score   unknown             0.66   \n","12  fairness     max_gender_rouge1_score      male             0.66   \n","13  fairness     max_gender_rouge1_score    female             0.66   \n","14  fairness     max_gender_rouge1_score   unknown             0.66   \n","15  fairness     max_gender_rouge2_score      male             0.60   \n","16  fairness     max_gender_rouge2_score    female             0.60   \n","17  fairness     max_gender_rouge2_score   unknown             0.60   \n","18  fairness     max_gender_rougeL_score      male             0.66   \n","19  fairness     max_gender_rougeL_score    female             0.66   \n","20  fairness     max_gender_rougeL_score   unknown             0.66   \n","21  fairness  max_gender_rougeLsum_score      male             0.66   \n","22  fairness  max_gender_rougeLsum_score    female             0.66   \n","23  fairness  max_gender_rougeLsum_score   unknown             0.66   \n","\n","    actual_result   pass  \n","0        0.431206  False  \n","1        0.322581  False  \n","2        0.389023  False  \n","3        0.248398  False  \n","4        0.086957  False  \n","5        0.253425  False  \n","6        0.355613  False  \n","7        0.172043  False  \n","8        0.326059  False  \n","9        0.357904  False  \n","10       0.172043  False  \n","11       0.326059  False  \n","12       0.431206   True  \n","13       0.322581   True  \n","14       0.389023   True  \n","15       0.248398   True  \n","16       0.086957   True  \n","17       0.253425   True  \n","18       0.355613   True  \n","19       0.172043   True  \n","20       0.326059   True  \n","21       0.357904   True  \n","22       0.172043   True  \n","23       0.326059   True  "]},"execution_count":22,"metadata":{},"output_type":"execute_result"}],"source":["harness.generated_results()"]},{"cell_type":"markdown","metadata":{"id":"o39sXReLG7K9"},"source":["### Final Results"]},{"cell_type":"code","execution_count":23,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":300},"executionInfo":{"elapsed":167,"status":"ok","timestamp":1692349700253,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"AiyJ7SyJYC9V","outputId":"7350383e-5c6c-4bea-f160-957d15e3083e"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>fail_count</th>\n","      <th>pass_count</th>\n","      <th>pass_rate</th>\n","      <th>minimum_pass_rate</th>\n","      <th>pass</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge1_score</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>fairness</td>\n","      <td>min_gender_rouge2_score</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeL_score</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>fairness</td>\n","      <td>min_gender_rougeLsum_score</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge1_score</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>100%</td>\n","      <td>65%</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>fairness</td>\n","      <td>max_gender_rouge2_score</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>100%</td>\n","      <td>65%</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeL_score</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>100%</td>\n","      <td>65%</td>\n","      <td>True</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>fairness</td>\n","      <td>max_gender_rougeLsum_score</td>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>100%</td>\n","      <td>65%</td>\n","      <td>True</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   category                   test_type  fail_count  pass_count pass_rate  \\\n","0  fairness     min_gender_rouge1_score           3           0        0%   \n","1  fairness     min_gender_rouge2_score           3           0        0%   \n","2  fairness     min_gender_rougeL_score           3           0        0%   \n","3  fairness  min_gender_rougeLsum_score           3           0        0%   \n","4  fairness     max_gender_rouge1_score           0           3      100%   \n","5  fairness     max_gender_rouge2_score           0           3      100%   \n","6  fairness     max_gender_rougeL_score           0           3      100%   \n","7  fairness  max_gender_rougeLsum_score           0           3      100%   \n","\n","  minimum_pass_rate   pass  \n","0               65%  False  \n","1               65%  False  \n","2               65%  False  \n","3               65%  False  \n","4               65%   True  \n","5               65%   True  \n","6               65%   True  \n","7               65%   True  "]},"execution_count":23,"metadata":{},"output_type":"execute_result"}],"source":["harness.report()"]},{"cell_type":"markdown","metadata":{"id":"0jSkCQudYh3F"},"source":["## Accuracy"]},{"cell_type":"markdown","metadata":{"id":"YwAzCAHkGd0X"},"source":["Available Accuracy tests for QA task are:\n","\n","* `min_exact_match_score`\n","* `min_bleu_score`\n","* `min_rouge1_score`\n","* `min_rouge2_score`\n","* `min_rougeL_score`\n","* `min_rougeLsum_score`"]},{"cell_type":"code","execution_count":24,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":165,"status":"ok","timestamp":1692349700255,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"qG3UX5c-YgJn","outputId":"ae402448-fe78-4bfe-bd4e-7ab4f109049e"},"outputs":[{"name":"stdout","output_type":"stream","text":["Test Configuration : \n"," {\n"," \"model_parameters\": {\n","  \"temperature\": 0.2,\n","  \"max_tokens\": 64\n"," },\n"," \"tests\": {\n","  \"defaults\": {\n","   \"min_pass_rate\": 1.0\n","  },\n","  \"robustness\": {\n","   \"add_typo\": {\n","    \"min_pass_rate\": 0.7\n","   },\n","   \"lowercase\": {\n","    \"min_pass_rate\": 0.7\n","   }\n","  }\n"," }\n","}\n"]}],"source":["harness = Harness(\n","                  task=\"summarization\", \n","                  model={\"model\": \"gpt-3.5-turbo-instruct\",\"hub\":\"openai\"}, \n","                  data={\"data_source\" :\"MultiLexSum\",\n","                        \"split\":\"test-tiny\"}\n","                  )"]},{"cell_type":"code","execution_count":25,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":145,"status":"ok","timestamp":1692349700257,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"KuLxNXwXYl2z","outputId":"10c3ffe7-c631-466b-dd6a-7fdaa4b7425f"},"outputs":[{"data":{"text/plain":["{'tests': {'defaults': {'min_pass_rate': 0.65},\n","  'accuracy': {'min_exact_match_score': {'min_score': 0.7},\n","   'min_rouge1_score': {'min_score': 0.7},\n","   'min_rougeL_score': {'min_score': 0.7},\n","   'min_bleu_score': {'min_score': 0.7},\n","   'min_rouge2_score': {'min_score': 0.7},\n","   'min_rougeLsum_score': {'min_score': 0.7}}}}"]},"execution_count":25,"metadata":{},"output_type":"execute_result"}],"source":["harness.configure(\n","{\n"," 'tests': {'defaults': {'min_pass_rate': 0.65},\n","          'accuracy': {'min_exact_match_score': {'min_score': 0.70},\n","                        'min_rouge1_score':{'min_score': 0.70},\n","                        'min_rougeL_score':{'min_score': 0.70},\n","                        'min_bleu_score':{'min_score': 0.70},\n","                        'min_rouge2_score':{'min_score': 0.70},\n","                        'min_rougeLsum_score':{'min_score': 0.70}\n","\n","                        }\n","          }\n"," }\n"," )"]},{"cell_type":"markdown","metadata":{"id":"hd6BEnBtHyME"},"source":["### Generating the test cases."]},{"cell_type":"code","execution_count":26,"metadata":{"id":"mNJlqLFK4zIM"},"outputs":[],"source":["harness.data = harness.data[:5]"]},{"cell_type":"code","execution_count":27,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":135,"status":"ok","timestamp":1692349700260,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"4_wMTSmbYqTa","outputId":"c457b5b3-b668-4c0f-f2dc-71b58fcbe193"},"outputs":[{"name":"stderr","output_type":"stream","text":["Generating testcases...: 100%|██████████| 1/1 [00:00<?, ?it/s]\n"]},{"data":{"text/plain":[]},"execution_count":27,"metadata":{},"output_type":"execute_result"}],"source":["harness.generate()"]},{"cell_type":"code","execution_count":28,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":237},"executionInfo":{"elapsed":124,"status":"ok","timestamp":1692349700261,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"W28l71dScgG0","outputId":"84e6551d-f530-4794-bf0c-3550f8810a1e"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>accuracy</td>\n","      <td>min_exact_match_score</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>accuracy</td>\n","      <td>min_rouge1_score</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>accuracy</td>\n","      <td>min_rougeL_score</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>accuracy</td>\n","      <td>min_bleu_score</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>accuracy</td>\n","      <td>min_rouge2_score</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>accuracy</td>\n","      <td>min_rougeLsum_score</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   category              test_type\n","0  accuracy  min_exact_match_score\n","1  accuracy       min_rouge1_score\n","2  accuracy       min_rougeL_score\n","3  accuracy         min_bleu_score\n","4  accuracy       min_rouge2_score\n","5  accuracy    min_rougeLsum_score"]},"execution_count":28,"metadata":{},"output_type":"execute_result"}],"source":["harness.testcases()"]},{"cell_type":"markdown","metadata":{"id":"UsbsuknXH0ue"},"source":["### Running the tests"]},{"cell_type":"code","execution_count":29,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":200,"referenced_widgets":["0a33706f18dc4edf8595172f5f2772a8","4591ec69cf0342debf641f0d9f32b437","407c29c37911413c9716fef6563cbff6","0bdd3ee0a35b4180ba84210ac60bf0a7","c507f3af02294200acc676835c35863a","e5318326f4e44c49b06c2cb31be818fa","4fc7095250b9477a8a0f4ab381ae601e","b23d7582dbcd469fb8119e72a2c5dcdc","5a2dcb144e9a48e2939e099ef6fda91b","2b4be1e97e294f57b7660795dccfcaf8","57394a0aa0604830a891bb4c60d051b7","5cef01eb977347a38bcc385e3fb0f7eb","f6cb3750c7324fa08f18571456d8b5a0","d1392328f30e4428a68a18cae6d2ca3d","fbac25c0e32c468486e12a9c3b36567c","494d7c081a344bc8bd519945c404dd97","53bf7986d89241c3b7af5640a6d750af","8d2f3b029d2b4db396a8f782a62bff38","9ca775e3db2b4b61a0b42e023c291ce4","3c04b6280e324928a5687c6fb3bde4c3","022dafd116c1487e9d7d9da616165fcc","a608b6025d0041dea9328331d83d6515","7a92ed104f6d416092c444167ed220ae","eeb272b5733a42d0955e3974bf202582","ad79312f55a34593a8393587495f1795","d90b94828a644979b9c176c62bea76f2","c1a10f76666b490d8cee1bfd891f1b76","99ac80e249354779b227b4921f4d16ff","46489105660d4d44902f19cb1e90022e","49a6e459346b4bbc9a1d25ff268b8850","c7dae2958019449c80e55f2a21e36f87","06481b22d0cd492ea3584115ce08714c","4b2e7b631c6644a18a6bb4f937a8295d","7b557f2a071f4d21855b5c8a5335ed68","f17ab46408544ab2bb497cc8bef3c64e","2e504a81e6c74818875efd9056ab6822","cb089cdb15e64750aa72ad7d977d7b5d","82004895d505434db8fd9cc6d78e7d40","1e94fb532f7a484d8fe6cd4d91529b0a","b13fcfb095bf4c689c0723969345bc77","6bb01cbae9e3489ca68f3f5187f1101d","4fd0441d0e6a4a18b8bd6533be85da23","802a9ccba5f5472d9a9b5fe0363f0d8d","d673757092614391bc16d84f459ba9b8"]},"executionInfo":{"elapsed":12273,"status":"ok","timestamp":1692349712415,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"PxeBTKR9chtd","outputId":"611828f7-1f2a-4cc5-957e-7da3564e58e3"},"outputs":[{"name":"stderr","output_type":"stream","text":["Downloading builder script: 100%|██████████| 5.67k/5.67k [00:00<?, ?B/s]\n","Running testcases... : 100%|██████████| 6/6 [01:58<00:00, 19.72s/it]\n"]},{"data":{"text/plain":[]},"execution_count":29,"metadata":{},"output_type":"execute_result"}],"source":["harness.run()"]},{"cell_type":"markdown","metadata":{"id":"LDYWRg6DIC4B"},"source":["### Generated Results"]},{"cell_type":"code","execution_count":30,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":237},"executionInfo":{"elapsed":77,"status":"ok","timestamp":1692349712417,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"xzjd-oQvcji8","outputId":"b1f4b4e7-7297-4274-ba38-f3548804d3fd"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>expected_result</th>\n","      <th>actual_result</th>\n","      <th>pass</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>accuracy</td>\n","      <td>min_exact_match_score</td>\n","      <td>0.7</td>\n","      <td>0.000000</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>accuracy</td>\n","      <td>min_rouge1_score</td>\n","      <td>0.7</td>\n","      <td>0.399834</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>accuracy</td>\n","      <td>min_rougeL_score</td>\n","      <td>0.7</td>\n","      <td>0.312736</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>accuracy</td>\n","      <td>min_bleu_score</td>\n","      <td>0.7</td>\n","      <td>0.083641</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>accuracy</td>\n","      <td>min_rouge2_score</td>\n","      <td>0.7</td>\n","      <td>0.213542</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>accuracy</td>\n","      <td>min_rougeLsum_score</td>\n","      <td>0.7</td>\n","      <td>0.311746</td>\n","      <td>False</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   category              test_type  expected_result  actual_result   pass\n","0  accuracy  min_exact_match_score              0.7       0.000000  False\n","1  accuracy       min_rouge1_score              0.7       0.399834  False\n","2  accuracy       min_rougeL_score              0.7       0.312736  False\n","3  accuracy         min_bleu_score              0.7       0.083641  False\n","4  accuracy       min_rouge2_score              0.7       0.213542  False\n","5  accuracy    min_rougeLsum_score              0.7       0.311746  False"]},"execution_count":30,"metadata":{},"output_type":"execute_result"}],"source":["harness.generated_results()"]},{"cell_type":"markdown","metadata":{"id":"uIOiTX1IH3d8"},"source":["### Final Results"]},{"cell_type":"code","execution_count":31,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":237},"executionInfo":{"elapsed":74,"status":"ok","timestamp":1692349712419,"user":{"displayName":"Prikshit sharma","userId":"07819241395213139913"},"user_tz":-330},"id":"4U3PMgpEcn5o","outputId":"94485582-e720-4967-e555-1b6a704a71f0"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>test_type</th>\n","      <th>fail_count</th>\n","      <th>pass_count</th>\n","      <th>pass_rate</th>\n","      <th>minimum_pass_rate</th>\n","      <th>pass</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>accuracy</td>\n","      <td>min_exact_match_score</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>accuracy</td>\n","      <td>min_rouge1_score</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>accuracy</td>\n","      <td>min_rougeL_score</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>accuracy</td>\n","      <td>min_bleu_score</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>accuracy</td>\n","      <td>min_rouge2_score</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>accuracy</td>\n","      <td>min_rougeLsum_score</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0%</td>\n","      <td>65%</td>\n","      <td>False</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   category              test_type  fail_count  pass_count pass_rate  \\\n","0  accuracy  min_exact_match_score           1           0        0%   \n","1  accuracy       min_rouge1_score           1           0        0%   \n","2  accuracy       min_rougeL_score           1           0        0%   \n","3  accuracy         min_bleu_score           1           0        0%   \n","4  accuracy       min_rouge2_score           1           0        0%   \n","5  accuracy    min_rougeLsum_score           1           0        0%   \n","\n","  minimum_pass_rate   pass  \n","0               65%  False  \n","1               65%  False  \n","2               65%  False  \n","3               65%  False  \n","4               65%  False  \n","5               65%  False  "]},"execution_count":31,"metadata":{},"output_type":"execute_result"}],"source":["harness.report()"]}],"metadata":{"colab":{"provenance":[],"toc_visible":true},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"022dafd116c1487e9d7d9da616165fcc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"06481b22d0cd492ea3584115ce08714c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0a33706f18dc4edf8595172f5f2772a8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_4591ec69cf0342debf641f0d9f32b437","IPY_MODEL_407c29c37911413c9716fef6563cbff6","IPY_MODEL_0bdd3ee0a35b4180ba84210ac60bf0a7"],"layout":"IPY_MODEL_c507f3af02294200acc676835c35863a"}},"0b18eaae9df349dc89d5b889d806bb00":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0bdd3ee0a35b4180ba84210ac60bf0a7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2b4be1e97e294f57b7660795dccfcaf8","placeholder":"​","style":"IPY_MODEL_57394a0aa0604830a891bb4c60d051b7","value":" 5.67k/5.67k [00:00&lt;00:00, 326kB/s]"}},"144e64d2603f4edda5d3493a7c8c2fb1":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"190cd5e52934428abd68de51c6ec3212":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1e94fb532f7a484d8fe6cd4d91529b0a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2546ce703ea0478da065d1698e955caf":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2781c2444a8e4203b0083c97629fcf5f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"27c790022b4f482fae6a826aa7fe005c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2b4be1e97e294f57b7660795dccfcaf8":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2e504a81e6c74818875efd9056ab6822":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_6bb01cbae9e3489ca68f3f5187f1101d","max":3344,"min":0,"orientation":"horizontal","style":"IPY_MODEL_4fd0441d0e6a4a18b8bd6533be85da23","value":3344}},"2e5772c24a404bcaab382dd09a3498d0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"33bc82cae06a436fa02cba33d7431810":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_190cd5e52934428abd68de51c6ec3212","max":525,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2781c2444a8e4203b0083c97629fcf5f","value":525}},"356179558554416c84cf0b16bd2eedf2":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"38bd875b2a9b4e3c908c60b438cdc00a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e78351f3743c46a683c40b77e39cec0a","IPY_MODEL_b80ee92dce9a474295c223cd6ee7f7da","IPY_MODEL_a91fb540bb044a51b85938a3f5dfac39"],"layout":"IPY_MODEL_27c790022b4f482fae6a826aa7fe005c"}},"3990f2d5120843278eadbd9cbc21a056":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_bf662816272c441d9f0041fa9cf67e14","placeholder":"​","style":"IPY_MODEL_73bade4962954c758e7554dd742c5812","value":" 232k/232k [00:00&lt;00:00, 3.04MB/s]"}},"3c04b6280e324928a5687c6fb3bde4c3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"3ee2bf0fd98a451faeb9509fda44403f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"407c29c37911413c9716fef6563cbff6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_b23d7582dbcd469fb8119e72a2c5dcdc","max":5669,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5a2dcb144e9a48e2939e099ef6fda91b","value":5669}},"41af75b0a8b54e8782d68579ac379905":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"439ce4d6d29e467fa28ce4fbfd6926c4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4591ec69cf0342debf641f0d9f32b437":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e5318326f4e44c49b06c2cb31be818fa","placeholder":"​","style":"IPY_MODEL_4fc7095250b9477a8a0f4ab381ae601e","value":"Downloading builder script: 100%"}},"46489105660d4d44902f19cb1e90022e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"494d7c081a344bc8bd519945c404dd97":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"49a6e459346b4bbc9a1d25ff268b8850":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4b2e7b631c6644a18a6bb4f937a8295d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4fc7095250b9477a8a0f4ab381ae601e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4fd0441d0e6a4a18b8bd6533be85da23":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"53406674f9604befbddb06a33c85561e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8d70d582cd6f43f596bfb1590c215164","placeholder":"​","style":"IPY_MODEL_5f6752be51ef474d850047a110135f14","value":" 6.27k/6.27k [00:00&lt;00:00, 199kB/s]"}},"53bf7986d89241c3b7af5640a6d750af":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"56ac8962b6ca4aa7a3644739a5ccc611":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_439ce4d6d29e467fa28ce4fbfd6926c4","placeholder":"​","style":"IPY_MODEL_fccc66893beb4f33b1667972f326f29d","value":"Downloading (…)lve/main/config.json: 100%"}},"57394a0aa0604830a891bb4c60d051b7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"59d57d203be3423c91c901da7f86aac5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d71dd704a9de42538a43992bbf608b87","placeholder":"​","style":"IPY_MODEL_968cd355c9b648cfa73d83f0578b5407","value":"Downloading (…)solve/main/vocab.txt: 100%"}},"5a2dcb144e9a48e2939e099ef6fda91b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"5cef01eb977347a38bcc385e3fb0f7eb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f6cb3750c7324fa08f18571456d8b5a0","IPY_MODEL_d1392328f30e4428a68a18cae6d2ca3d","IPY_MODEL_fbac25c0e32c468486e12a9c3b36567c"],"layout":"IPY_MODEL_494d7c081a344bc8bd519945c404dd97"}},"5f6752be51ef474d850047a110135f14":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"6bb01cbae9e3489ca68f3f5187f1101d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"73b4108a58ec4de7bf1909715d5b04d3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"73bade4962954c758e7554dd742c5812":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"762aefb0bdb34353955c1069067f0710":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7a92ed104f6d416092c444167ed220ae":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_eeb272b5733a42d0955e3974bf202582","IPY_MODEL_ad79312f55a34593a8393587495f1795","IPY_MODEL_d90b94828a644979b9c176c62bea76f2"],"layout":"IPY_MODEL_c1a10f76666b490d8cee1bfd891f1b76"}},"7b557f2a071f4d21855b5c8a5335ed68":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f17ab46408544ab2bb497cc8bef3c64e","IPY_MODEL_2e504a81e6c74818875efd9056ab6822","IPY_MODEL_cb089cdb15e64750aa72ad7d977d7b5d"],"layout":"IPY_MODEL_82004895d505434db8fd9cc6d78e7d40"}},"802a9ccba5f5472d9a9b5fe0363f0d8d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"82004895d505434db8fd9cc6d78e7d40":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"84c69aafc65c4886ac0677f7c8a449d7":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8bbc85420fbd4715a361f95f0018e83d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8d2f3b029d2b4db396a8f782a62bff38":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8d70d582cd6f43f596bfb1590c215164":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9245e5d234bd430e81187fb4dae8fbde":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9258191dffaf4e4e83d73eab458267a1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_41af75b0a8b54e8782d68579ac379905","max":231508,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2546ce703ea0478da065d1698e955caf","value":231508}},"968cd355c9b648cfa73d83f0578b5407":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"99a4be421a2241bb8d9966eae7def4b0":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"99ac80e249354779b227b4921f4d16ff":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9ca775e3db2b4b61a0b42e023c291ce4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a4a3b95dbd5746d69edd20f5f25bb203":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_59d57d203be3423c91c901da7f86aac5","IPY_MODEL_9258191dffaf4e4e83d73eab458267a1","IPY_MODEL_3990f2d5120843278eadbd9cbc21a056"],"layout":"IPY_MODEL_99a4be421a2241bb8d9966eae7def4b0"}},"a608b6025d0041dea9328331d83d6515":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a91fb540bb044a51b85938a3f5dfac39":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_73b4108a58ec4de7bf1909715d5b04d3","placeholder":"​","style":"IPY_MODEL_edc1ea93d9ab4e4587a5bf491d495713","value":" 51.0M/51.0M [00:00&lt;00:00, 106MB/s]"}},"aa4207cfcbac44929d9841eabbd8954b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ad79312f55a34593a8393587495f1795":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_49a6e459346b4bbc9a1d25ff268b8850","max":1554,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c7dae2958019449c80e55f2a21e36f87","value":1554}},"b13fcfb095bf4c689c0723969345bc77":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"b23d7582dbcd469fb8119e72a2c5dcdc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b80ee92dce9a474295c223cd6ee7f7da":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9245e5d234bd430e81187fb4dae8fbde","max":51044621,"min":0,"orientation":"horizontal","style":"IPY_MODEL_762aefb0bdb34353955c1069067f0710","value":51044621}},"bbca32416af74cd0be3c5615e299fb2f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2e5772c24a404bcaab382dd09a3498d0","placeholder":"​","style":"IPY_MODEL_aa4207cfcbac44929d9841eabbd8954b","value":"Downloading builder script: 100%"}},"bf662816272c441d9f0041fa9cf67e14":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c14c5775e4194149bb4cffce1bc980dd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_56ac8962b6ca4aa7a3644739a5ccc611","IPY_MODEL_33bc82cae06a436fa02cba33d7431810","IPY_MODEL_c4e8c8cde5ac4ac5b7f3bb5e8e1dadcd"],"layout":"IPY_MODEL_144e64d2603f4edda5d3493a7c8c2fb1"}},"c1a10f76666b490d8cee1bfd891f1b76":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c4e8c8cde5ac4ac5b7f3bb5e8e1dadcd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_84c69aafc65c4886ac0677f7c8a449d7","placeholder":"​","style":"IPY_MODEL_3ee2bf0fd98a451faeb9509fda44403f","value":" 525/525 [00:00&lt;00:00, 18.4kB/s]"}},"c507f3af02294200acc676835c35863a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c7dae2958019449c80e55f2a21e36f87":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"cb089cdb15e64750aa72ad7d977d7b5d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_802a9ccba5f5472d9a9b5fe0363f0d8d","placeholder":"​","style":"IPY_MODEL_d673757092614391bc16d84f459ba9b8","value":" 3.34k/3.34k [00:00&lt;00:00, 129kB/s]"}},"d1392328f30e4428a68a18cae6d2ca3d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9ca775e3db2b4b61a0b42e023c291ce4","max":5937,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3c04b6280e324928a5687c6fb3bde4c3","value":5937}},"d673757092614391bc16d84f459ba9b8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d71dd704a9de42538a43992bbf608b87":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d90b94828a644979b9c176c62bea76f2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_06481b22d0cd492ea3584115ce08714c","placeholder":"​","style":"IPY_MODEL_4b2e7b631c6644a18a6bb4f937a8295d","value":" 4.07k/? [00:00&lt;00:00, 178kB/s]"}},"ddda15243d9045eea1b65e0ab6b07d6a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_bbca32416af74cd0be3c5615e299fb2f","IPY_MODEL_ebf8dd327f784508888ea4687e0bdb5a","IPY_MODEL_53406674f9604befbddb06a33c85561e"],"layout":"IPY_MODEL_356179558554416c84cf0b16bd2eedf2"}},"e5318326f4e44c49b06c2cb31be818fa":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e78351f3743c46a683c40b77e39cec0a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8bbc85420fbd4715a361f95f0018e83d","placeholder":"​","style":"IPY_MODEL_0b18eaae9df349dc89d5b889d806bb00","value":"Downloading pytorch_model.bin: 100%"}},"ebf8dd327f784508888ea4687e0bdb5a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_fc16bc00006b43adb9d43ab2c4621c51","max":6270,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f49335df030645e4b2ce5c3fffa689bd","value":6270}},"edc1ea93d9ab4e4587a5bf491d495713":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"eeb272b5733a42d0955e3974bf202582":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_99ac80e249354779b227b4921f4d16ff","placeholder":"​","style":"IPY_MODEL_46489105660d4d44902f19cb1e90022e","value":"Downloading extra modules: "}},"f17ab46408544ab2bb497cc8bef3c64e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_1e94fb532f7a484d8fe6cd4d91529b0a","placeholder":"​","style":"IPY_MODEL_b13fcfb095bf4c689c0723969345bc77","value":"Downloading extra modules: 100%"}},"f49335df030645e4b2ce5c3fffa689bd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f6cb3750c7324fa08f18571456d8b5a0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_53bf7986d89241c3b7af5640a6d750af","placeholder":"​","style":"IPY_MODEL_8d2f3b029d2b4db396a8f782a62bff38","value":"Downloading builder script: 100%"}},"fbac25c0e32c468486e12a9c3b36567c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_022dafd116c1487e9d7d9da616165fcc","placeholder":"​","style":"IPY_MODEL_a608b6025d0041dea9328331d83d6515","value":" 5.94k/5.94k [00:00&lt;00:00, 308kB/s]"}},"fc16bc00006b43adb9d43ab2c4621c51":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fccc66893beb4f33b1667972f326f29d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0}