examples/experiments/Robust04.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TREC Robust 04\n",
    "\n",
    "This notebook demonstrates baseline experiments on the TREC Robust04 test collection. More information is provided [in the PyTerrier documenatation](https://pyterrier.readthedocs.io/en/latest/experiments/Robust04.html)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install PyTerrier - this installs the latest version from the GitHub repository."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T10:55:05.912053Z",
     "iopub.status.busy": "2020-12-17T10:55:05.911422Z",
     "iopub.status.idle": "2020-12-17T10:55:05.917004Z",
     "shell.execute_reply": "2020-12-17T10:55:05.915944Z",
     "shell.execute_reply.started": "2020-12-17T10:55:05.911784Z"
    }
   },
   "outputs": [],
   "source": [
    "!pip install python-terrier\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start PyTerrier. By using `version='snapshot'`, we use the latest version of Terrier from its own [GitHub repository](http://github.com/terrier-org/terrier-core/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T10:55:05.919430Z",
     "iopub.status.busy": "2020-12-17T10:55:05.918891Z",
     "iopub.status.idle": "2020-12-17T10:57:17.175579Z",
     "shell.execute_reply": "2020-12-17T10:57:17.174485Z",
     "shell.execute_reply.started": "2020-12-17T10:55:05.919250Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading terrier-assemblies 5.x-SNAPSHOT  jar-with-dependencies to /users/craigm/.pyterrier...\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "import pyterrier as pt\n",
    "if not pt.started():\n",
    "    pt.init(mem=8000, version='snapshot', tqdm='notebook', \n",
    "            boot_packages=[\"com.github.terrierteam:terrier-prf:-SNAPSHOT\"]\n",
    "           )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Update this configuration, to detail:\n",
    " - where your copy of the TREC Disk 4 and 5 corpus is\n",
    " - where you wish to store your index."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T10:57:17.177656Z",
     "iopub.status.busy": "2020-12-17T10:57:17.177387Z",
     "iopub.status.idle": "2020-12-17T10:57:17.181543Z",
     "shell.execute_reply": "2020-12-17T10:57:17.180703Z",
     "shell.execute_reply.started": "2020-12-17T10:57:17.177600Z"
    }
   },
   "outputs": [],
   "source": [
    "DISK45_PATH=\"/local/collections/TRECdisk45/\"\n",
    "INDEX_DIR=\"/local/indices/disk45\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indexing\n",
    "\n",
    "This indexes the corpus; it took around 8 minutes using a single thread."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T11:11:28.067115Z",
     "iopub.status.busy": "2020-12-17T11:11:28.066661Z",
     "iopub.status.idle": "2020-12-17T11:19:24.639919Z",
     "shell.execute_reply": "2020-12-17T11:19:24.638981Z",
     "shell.execute_reply.started": "2020-12-17T11:11:28.067040Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:11:28.315 [main] WARN  o.t.i.MultiDocumentFileCollection - trec.encoding is not set; resorting to platform default (ISO-8859-1). Indexing may be platform dependent. Recommend trec.encoding=UTF-8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2744b78d92e44ec8a7480caeb86b22d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2299.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Number of documents: 528155\n",
      "Number of terms: 738439\n",
      "Number of fields: 0\n",
      "Number of tokens: 156321446\n",
      "Field names: []\n",
      "Positions:   false\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "if os.path.exists(os.path.join(INDEX_DIR, \"data.properties\")):\n",
    "    indexref = pt.IndexRef.of(os.path.join(INDEX_DIR, \"data.properties\"))\n",
    "else:    \n",
    "    files = pt.io.find_files(DISK45_PATH)\n",
    "    # no-one indexes the congressional record in directory /CR/\n",
    "    # indeed, recent copies from NIST dont contain it\n",
    "    # we also remove some of the other unneeded files\n",
    "    bad = ['/CR/', '/AUX/', 'READCHG', 'READMEFB', 'READFRCG', 'READMEFR', 'READMEFT', 'READMELA']\n",
    "    for b in bad:\n",
    "        files = list(filter(lambda f: b not in f, files))\n",
    "    indexer = pt.TRECCollectionIndexer(INDEX_DIR, verbose=True)\n",
    "    indexref = indexer.index(files)\n",
    "    # processing the files took 7 minutes; the total indexing process took 7m40\n",
    "\n",
    "index = pt.IndexFactory.of(indexref)\n",
    "print(index.getCollectionStatistics().toString())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieval - Simple Weighting Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:23:17.131341Z",
     "iopub.status.busy": "2020-12-17T13:23:17.130720Z",
     "iopub.status.idle": "2020-12-17T13:23:17.169530Z",
     "shell.execute_reply": "2020-12-17T13:23:17.168761Z",
     "shell.execute_reply.started": "2020-12-17T13:23:17.131281Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "BM25 = pt.terrier.Retriever(index, wmodel=\"BM25\")\n",
    "DPH  = pt.terrier.Retriever(index, wmodel=\"DPH\")\n",
    "PL2  = pt.terrier.Retriever(index, wmodel=\"PL2\")\n",
    "DLM  = pt.terrier.Retriever(index, wmodel=\"DirichletLM\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:24:04.056101Z",
     "iopub.status.busy": "2020-12-17T13:24:04.055681Z",
     "iopub.status.idle": "2020-12-17T13:24:51.380289Z",
     "shell.execute_reply": "2020-12-17T13:24:51.379190Z",
     "shell.execute_reply.started": "2020-12-17T13:24:04.056037Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13:24:04.061 [main] WARN  o.t.a.batchquerying.TRECQuery - trec.encoding is not set; resorting to platform default (ISO-8859-1). Retrieval may be platform dependent. Recommend trec.encoding=UTF-8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>map</th>\n",
       "      <th>P_10</th>\n",
       "      <th>P_20</th>\n",
       "      <th>ndcg_cut_20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM25</td>\n",
       "      <td>0.241763</td>\n",
       "      <td>0.426104</td>\n",
       "      <td>0.349398</td>\n",
       "      <td>0.408061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DPH</td>\n",
       "      <td>0.251307</td>\n",
       "      <td>0.447390</td>\n",
       "      <td>0.361446</td>\n",
       "      <td>0.422524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PL2</td>\n",
       "      <td>0.229386</td>\n",
       "      <td>0.420884</td>\n",
       "      <td>0.343775</td>\n",
       "      <td>0.402179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dirichlet QL</td>\n",
       "      <td>0.236826</td>\n",
       "      <td>0.407631</td>\n",
       "      <td>0.337952</td>\n",
       "      <td>0.396870</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           name       map      P_10      P_20  ndcg_cut_20\n",
       "0          BM25  0.241763  0.426104  0.349398     0.408061\n",
       "1           DPH  0.251307  0.447390  0.361446     0.422524\n",
       "2           PL2  0.229386  0.420884  0.343775     0.402179\n",
       "3  Dirichlet QL  0.236826  0.407631  0.337952     0.396870"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pt.Experiment(\n",
    "    [BM25, DPH, PL2, DLM],\n",
    "    pt.get_dataset(\"trec-robust-2004\").get_topics(),\n",
    "    pt.get_dataset(\"trec-robust-2004\").get_qrels(),\n",
    "    eval_metrics=[\"map\", \"P_10\", \"P_20\", \"ndcg_cut_20\"],\n",
    "    names=[\"BM25\", \"DPH\", \"PL2\", \"Dirichlet QL\"]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieval - Query Expansion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:24:51.382831Z",
     "iopub.status.busy": "2020-12-17T13:24:51.382329Z",
     "iopub.status.idle": "2020-12-17T13:24:51.433914Z",
     "shell.execute_reply": "2020-12-17T13:24:51.433212Z",
     "shell.execute_reply.started": "2020-12-17T13:24:51.382660Z"
    }
   },
   "outputs": [],
   "source": [
    "Bo1 = pt.rewrite.Bo1QueryExpansion(index)\n",
    "KL = pt.rewrite.KLQueryExpansion(index)\n",
    "RM3 = pt.rewrite.RM3(index)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:24:51.436346Z",
     "iopub.status.busy": "2020-12-17T13:24:51.435857Z",
     "iopub.status.idle": "2020-12-17T13:26:53.528275Z",
     "shell.execute_reply": "2020-12-17T13:26:53.527534Z",
     "shell.execute_reply.started": "2020-12-17T13:24:51.436189Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13:24:51.441 [main] WARN  o.t.a.batchquerying.TRECQuery - trec.encoding is not set; resorting to platform default (ISO-8859-1). Retrieval may be platform dependent. Recommend trec.encoding=UTF-8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>map</th>\n",
       "      <th>P_10</th>\n",
       "      <th>P_20</th>\n",
       "      <th>ndcg_cut_20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM25</td>\n",
       "      <td>0.241763</td>\n",
       "      <td>0.426104</td>\n",
       "      <td>0.349398</td>\n",
       "      <td>0.408061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>+Bo1</td>\n",
       "      <td>0.279458</td>\n",
       "      <td>0.448996</td>\n",
       "      <td>0.378916</td>\n",
       "      <td>0.436533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>+KL</td>\n",
       "      <td>0.279401</td>\n",
       "      <td>0.444177</td>\n",
       "      <td>0.378313</td>\n",
       "      <td>0.435196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>+RM3</td>\n",
       "      <td>0.276544</td>\n",
       "      <td>0.453815</td>\n",
       "      <td>0.379518</td>\n",
       "      <td>0.430367</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name       map      P_10      P_20  ndcg_cut_20\n",
       "0  BM25  0.241763  0.426104  0.349398     0.408061\n",
       "1  +Bo1  0.279458  0.448996  0.378916     0.436533\n",
       "2   +KL  0.279401  0.444177  0.378313     0.435196\n",
       "3  +RM3  0.276544  0.453815  0.379518     0.430367"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pt.Experiment(\n",
    "    [\n",
    "            BM25, \n",
    "            BM25 >> Bo1 >> BM25, \n",
    "            BM25 >> KL >> BM25, \n",
    "            BM25 >> RM3 >> BM25, \n",
    "    ],\n",
    "    pt.get_dataset(\"trec-robust-2004\").get_topics(),\n",
    "    pt.get_dataset(\"trec-robust-2004\").get_qrels(),\n",
    "    eval_metrics=[\"map\", \"P_10\", \"P_20\", \"ndcg_cut_20\"],\n",
    "    names=[\"BM25\", \"+Bo1\", \"+KL\", \"+RM3\"]\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-17T13:26:53.529721Z",
     "iopub.status.busy": "2020-12-17T13:26:53.529495Z",
     "iopub.status.idle": "2020-12-17T13:29:00.956677Z",
     "shell.execute_reply": "2020-12-17T13:29:00.955944Z",
     "shell.execute_reply.started": "2020-12-17T13:26:53.529682Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13:26:53.533 [main] WARN  o.t.a.batchquerying.TRECQuery - trec.encoding is not set; resorting to platform default (ISO-8859-1). Retrieval may be platform dependent. Recommend trec.encoding=UTF-8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>map</th>\n",
       "      <th>P_10</th>\n",
       "      <th>P_20</th>\n",
       "      <th>ndcg_cut_20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DPH</td>\n",
       "      <td>0.251307</td>\n",
       "      <td>0.447390</td>\n",
       "      <td>0.361446</td>\n",
       "      <td>0.422524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>+Bo1</td>\n",
       "      <td>0.285334</td>\n",
       "      <td>0.458635</td>\n",
       "      <td>0.387952</td>\n",
       "      <td>0.444528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>+KL</td>\n",
       "      <td>0.285720</td>\n",
       "      <td>0.458635</td>\n",
       "      <td>0.386948</td>\n",
       "      <td>0.442636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>+RM3</td>\n",
       "      <td>0.281796</td>\n",
       "      <td>0.461044</td>\n",
       "      <td>0.389960</td>\n",
       "      <td>0.441863</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   name       map      P_10      P_20  ndcg_cut_20\n",
       "0   DPH  0.251307  0.447390  0.361446     0.422524\n",
       "1  +Bo1  0.285334  0.458635  0.387952     0.444528\n",
       "2   +KL  0.285720  0.458635  0.386948     0.442636\n",
       "3  +RM3  0.281796  0.461044  0.389960     0.441863"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pt.Experiment(\n",
    "    [\n",
    "            DPH, \n",
    "            DPH >> Bo1 >> DPH, \n",
    "            DPH >> KL >> DPH, \n",
    "            DPH >> RM3 >> DPH, \n",
    "    ],\n",
    "    pt.get_dataset(\"trec-robust-2004\").get_topics(),\n",
    "    pt.get_dataset(\"trec-robust-2004\").get_qrels(),\n",
    "    eval_metrics=[\"map\", \"P_10\", \"P_20\", \"ndcg_cut_20\"],\n",
    "    names=[\"DPH\", \"+Bo1\", \"+KL\", \"+RM3\"]\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyterrier",
   "language": "python",
   "name": "pyterrier"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}