09-optimize-term-similarity-matrices.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "445c1cfb-f4d9-4481-ae38-ccfcb4ba2aa7",
   "metadata": {},
   "source": [
    "# Optimize token similarity matrices\n",
    "\n",
    "In this notebook, we will optimize the hyperparameters of token similarity matrices for our best run from the previous notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "30d4db92-de63-45d6-aea5-fb5bf1f203cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mir\n"
     ]
    }
   ],
   "source": [
    "! hostname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "472ba006-5dc6-4fbd-b20c-c1690b496529",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "! pip install .[system]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f89f5d58-c62b-49d8-a950-b570a5a776bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "\n",
    "from pandas import DataFrame\n",
    "\n",
    "def evaluate_interpolated_run(basename: str) -> DataFrame:\n",
    "    match = re.search('(?P<symmetric>(True|False))-(?P<dominant>(True|False))-(?P<nonzero_limit>([0-9]+))$', basename)\n",
    "    if not match:\n",
    "        symmetric = 'True'\n",
    "        dominant = 'True for Levenshtein, False for word embeddings'\n",
    "        nonzero_limit = '100'\n",
    "    else:\n",
    "        symmetric = match.group('symmetric')\n",
    "        dominant = match.group('dominant')\n",
    "        nonzero_limit = match.group('nonzero_limit')\n",
    "    \n",
    "    with open(f'submission/{basename}.first_alpha_and_gamma', 'rt') as f:\n",
    "        first_alpha_and_gamma = json.load(f)\n",
    "    if 'alpha' in first_alpha_and_gamma or 'gamma' in first_alpha_and_gamma:\n",
    "        raise ValueError(f'First system from run {basename} is not yet optimized')\n",
    "\n",
    "    first_alpha = first_alpha_and_gamma['best_alpha']\n",
    "    first_gamma = first_alpha_and_gamma['best_gamma']\n",
    "\n",
    "    with open(f'submission/{basename}.second_alpha_and_gamma', 'rt') as f:\n",
    "        second_alpha_and_gamma = json.load(f)\n",
    "    if 'alpha' in second_alpha_and_gamma or 'gamma' in second_alpha_and_gamma:\n",
    "        raise ValueError(f'Second system from run {basename} is not yet optimized')\n",
    "\n",
    "    second_alpha = second_alpha_and_gamma['best_alpha']\n",
    "    second_gamma = second_alpha_and_gamma['best_gamma']\n",
    "\n",
    "    with open(f'submission/{basename}.beta', 'rt') as f:\n",
    "        _beta = json.load(f)\n",
    "    if 'beta' in _beta:\n",
    "        raise ValueError(f'Interpolated system from run {basename} is not yet optimized')\n",
    "    \n",
    "    beta = _beta['best_beta']\n",
    "    \n",
    "    with open(f'submission/{basename}.ndcg_score', 'rt') as f:\n",
    "        ndcg = f.read()\n",
    "\n",
    "    ndcg, *_ = ndcg.split(', ')\n",
    "    ndcg = float(ndcg)\n",
    "        \n",
    "    formatters = {\"first_alpha\": lambda alpha: f'{alpha:.1f}',\n",
    "                  \"first_gamma\": lambda gamma: f'{gamma:g}',\n",
    "                  \"second_alpha\": lambda alpha: f'{alpha:.1f}',\n",
    "                  \"second_gamma\": lambda gamma: f'{gamma:g}',\n",
    "                  \"beta\": lambda beta: f'{beta:.1f}',\n",
    "                  \"ndcg\": lambda ndcg: f'{alpha:.3f}'}\n",
    "\n",
    "    rows = 'ARQMath-3',\n",
    "    columns = 'α₁', 'γ₁', 'α₂', 'γ₂', 'β', 'symmetric', 'dominant', 'nonzero_limit', \"NDCG'\"\n",
    "    data = [[first_alpha, first_gamma, second_alpha, second_gamma, beta, symmetric, dominant, nonzero_limit, ndcg]]\n",
    "\n",
    "    dataframe = DataFrame(data, columns=columns, index=rows)\n",
    "\n",
    "    return dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "322edf0c-520e-4621-9d81-8e00a0dfb8bc",
   "metadata": {},
   "source": [
    "## Results with hand-picked hyperparameters\n",
    "\n",
    "First, we show the results with hand-picked hyperparameters of token similarity matrices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a8f80b5d-24a7-4cf2-97b7-38421376fcf7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>α₁</th>\n",
       "      <th>γ₁</th>\n",
       "      <th>α₂</th>\n",
       "      <th>γ₂</th>\n",
       "      <th>β</th>\n",
       "      <th>symmetric</th>\n",
       "      <th>dominant</th>\n",
       "      <th>nonzero_limit</th>\n",
       "      <th>NDCG'</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ARQMath-3</th>\n",
       "      <td>0.7</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.7</td>\n",
       "      <td>True</td>\n",
       "      <td>True for Levenshtein, False for word embeddings</td>\n",
       "      <td>100</td>\n",
       "      <td>0.355</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            α₁  γ₁   α₂  γ₂    β symmetric  \\\n",
       "ARQMath-3  0.7   2  0.0   5  0.7      True   \n",
       "\n",
       "                                                  dominant nonzero_limit  \\\n",
       "ARQMath-3  True for Levenshtein, False for word embeddings           100   \n",
       "\n",
       "           NDCG'  \n",
       "ARQMath-3  0.355  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_interpolated_run('SCM-task1-interpolated_positional_word2vec_text+tangentl-both-auto-P')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a9c01be6-824c-4404-b0fb-2b4d51360411",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "def get_best_hyperparameters(basename: str) -> str:\n",
    "    best_ndcg, best_basename = float('-inf'), None\n",
    "    for pathname in Path('submission').glob(f'{basename}-*.beta'):\n",
    "        with pathname.open('rt') as f:\n",
    "            results = json.load(f)\n",
    "        ndcg = results['best_ndcg']\n",
    "        if ndcg > best_ndcg:\n",
    "            best_ndcg = ndcg\n",
    "            best_basename = pathname.stem\n",
    "    assert best_basename is not None\n",
    "    return best_basename"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a06b3c21-6a03-4d7d-8235-2084d5769e71",
   "metadata": {},
   "source": [
    "## Results with optimized hyperparameters\n",
    "\n",
    "Next, we show the results with optimized hyperparameters of token similarity matrices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "10ddf1e5-4475-4977-a423-55b87062f984",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "! make optimized-best-runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e3a519dd-4af9-4a49-8c5a-e21b3eb06cb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "basename = get_best_hyperparameters('SCM-task1-interpolated_positional_word2vec_text+tangentl-both-auto-P')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b58650c0-682e-41c1-ba4b-3f5871aec9f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>α₁</th>\n",
       "      <th>γ₁</th>\n",
       "      <th>α₂</th>\n",
       "      <th>γ₂</th>\n",
       "      <th>β</th>\n",
       "      <th>symmetric</th>\n",
       "      <th>dominant</th>\n",
       "      <th>nonzero_limit</th>\n",
       "      <th>NDCG'</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ARQMath-3</th>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>50</td>\n",
       "      <td>0.355</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            α₁  γ₁   α₂  γ₂    β symmetric dominant nonzero_limit  NDCG'\n",
       "ARQMath-3  0.8   2  0.0   5  0.7     False    False            50  0.355"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_interpolated_run(basename)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0ee3631-4160-4a38-a31b-12ed721e978d",
   "metadata": {},
   "source": [
    "NDCG' shows that the soft vector space model is robust to parameter variations."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}