santa-2023-metric.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a13e0de2",
   "metadata": {
    "_cell_guid": "4cf02a6f-b7e9-4360-892d-b1a50793eb12",
    "_uuid": "2a1239f3-55fc-4dbb-9d3a-e90bfffa038c",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2023-12-18T23:21:57.080345Z",
     "iopub.status.busy": "2023-12-18T23:21:57.079840Z",
     "iopub.status.idle": "2023-12-18T23:21:58.233825Z",
     "shell.execute_reply": "2023-12-18T23:21:58.232294Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 1.163488,
     "end_time": "2023-12-18T23:21:58.237508",
     "exception": false,
     "start_time": "2023-12-18T23:21:57.074020",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\"\"\"Evaluation metric for Santa 2023.\"\"\"\n",
    "\n",
    "import pandas as pd\n",
    "from ast import literal_eval\n",
    "from dataclasses import dataclass\n",
    "from sympy.combinatorics import Permutation\n",
    "from typing import Dict, List\n",
    "\n",
    "\n",
    "class ParticipantVisibleError(Exception):\n",
    "    pass\n",
    "\n",
    "\n",
    "def score(\n",
    "        solution: pd.DataFrame,\n",
    "        submission: pd.DataFrame,\n",
    "        series_id_column_name: str,\n",
    "        moves_column_name: str,\n",
    "        puzzle_info_path: str,\n",
    ") -> float:\n",
    "    \"\"\"Santa 2023 evaluation metric.\n",
    "    Parameters\n",
    "    ----------\n",
    "    solution : pd.DataFrame\n",
    "    submission : pd.DataFrame\n",
    "    series_id_column_name : str\n",
    "    moves_column_name : str\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    total_num_moves : int\n",
    "    \"\"\"\n",
    "    if list(submission.columns) != [series_id_column_name, moves_column_name]:\n",
    "        raise ParticipantVisibleError(\n",
    "            f\"Submission must have columns {series_id_column_name} and {moves_column_name}.\"\n",
    "        )\n",
    "\n",
    "    puzzle_info = pd.read_csv(puzzle_info_path, index_col='puzzle_type')\n",
    "    total_num_moves = 0\n",
    "    for sol, sub in zip(solution.itertuples(), submission.itertuples()):\n",
    "        puzzle_id = getattr(sol, series_id_column_name)\n",
    "        assert puzzle_id == getattr(sub, series_id_column_name)\n",
    "        allowed_moves = literal_eval(puzzle_info.loc[sol.puzzle_type, 'allowed_moves'])\n",
    "        allowed_moves = {k: Permutation(v) for k, v in allowed_moves.items()}\n",
    "        puzzle = Puzzle(\n",
    "            puzzle_id=puzzle_id,\n",
    "            allowed_moves=allowed_moves,\n",
    "            solution_state=sol.solution_state.split(';'),\n",
    "            initial_state=sol.initial_state.split(';'),\n",
    "            num_wildcards=sol.num_wildcards,\n",
    "        )\n",
    "\n",
    "        # Score submission row\n",
    "        total_num_moves += score_puzzle(puzzle_id, puzzle, getattr(sub, moves_column_name))\n",
    "\n",
    "    return total_num_moves\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class Puzzle:\n",
    "    \"\"\"A permutation puzzle.\"\"\"\n",
    "\n",
    "    puzzle_id: str\n",
    "    allowed_moves: Dict[str, List[int]]\n",
    "    solution_state: List[str]\n",
    "    initial_state: List[str]\n",
    "    num_wildcards: int\n",
    "\n",
    "\n",
    "def score_puzzle(puzzle_id, puzzle, sub_solution):\n",
    "    \"\"\"Score the solution to a permutation puzzle.\"\"\"\n",
    "    # Apply submitted sequence of moves to the initial state, from left to right\n",
    "    moves = sub_solution.split('.')\n",
    "    state = puzzle.initial_state\n",
    "    for m in moves:\n",
    "        power = 1\n",
    "        if m[0] == \"-\":\n",
    "            m = m[1:]\n",
    "            power = -1\n",
    "        try:\n",
    "            p = puzzle.allowed_moves[m]\n",
    "        except KeyError:\n",
    "            raise ParticipantVisibleError(f\"{m} is not an allowed move for {puzzle_id}.\")\n",
    "        state = (p ** power)(state)\n",
    "\n",
    "    # Check that submitted moves solve puzzle\n",
    "    num_wrong_facelets = sum(not(s == t) for s, t in zip(puzzle.solution_state, state))\n",
    "    if num_wrong_facelets > puzzle.num_wildcards:\n",
    "        raise ParticipantVisibleError(f\"Submitted moves do not solve {puzzle_id}.\")\n",
    "\n",
    "    # The score for this instance is the total number of moves needed to solve the puzzle\n",
    "    return len(moves)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 7323271,
     "sourceId": 65704,
     "sourceType": "competition"
    }
   ],
   "dockerImageVersionId": 30458,
   "isGpuEnabled": false,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 15.828457,
   "end_time": "2023-12-18T23:21:59.168042",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-12-18T23:21:43.339585",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}