Dimensionality reduction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b0894c8c",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f48a8210",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from analysis.dim_reduction import Data\n",
    "from analysis.widgets import DataWidgets as dw\n",
    "from analysis.widgets import save_df_button\n",
    "from scipy.spatial.distance import cosine\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import matplotlib as mpl\n",
    "\n",
    "mpl.rcParams['figure.dpi'] = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5024611",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca9e6833",
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to the folder with the processed data\n",
    "path_to_data = 'demo_movies'\n",
    "\n",
    "# whether to use the entropy of transfer\n",
    "transfer_entropy = False\n",
    "\n",
    "# sessions params\n",
    "# key - unique id\n",
    "# path - path to session data\n",
    "# mouse - mouse id\n",
    "# condition (1/2 etc)\n",
    "# fps - frames per second\n",
    "dates = {\n",
    "    'mouseold1': {'path': 'mouseold/1',\n",
    "                  'mouse': '1',\n",
    "                  'condition': '1',\n",
    "                  'fps': 20},\n",
    "    'mouseold2': {'path': 'mouseold/2',\n",
    "                  'mouse': '1',\n",
    "                  'condition': '1',\n",
    "                  'fps': 20},\n",
    "    'mouseold3': {'path': 'mouseold/3',\n",
    "                  'mouse': '1',\n",
    "                  'condition': '1',\n",
    "                  'fps': 20},\n",
    "\n",
    "    'mouseX': {'path': 'mouse3/right_after',\n",
    "               'mouse': '1',\n",
    "               'condition': '2',\n",
    "               'fps': 15},\n",
    "}\n",
    "\n",
    "conditions_order = {\n",
    "    '1': ['1', '2'],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb50f2bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# data loading\n",
    "data_class = Data(path_to_data, dates, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a5ae34a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# calculating statistics\n",
    "data_class.get_data(transfer_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85c0bac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dimension reduction\n",
    "data, pca = data_class.data_reduction()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ec04357",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# visualization\n",
    "dw.show_result(data_class, conditions_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cde06ed0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tabular representation of results\n",
    "result = data.copy()\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8d374e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "dw.save(data_class, path_to_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8d471de",
   "metadata": {},
   "source": [
    "# stats all\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4817a31",
   "metadata": {},
   "outputs": [],
   "source": [
    "dw.show_stat(data_class, conditions_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63eba1aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "dw.stats_deviation(data_class, path_to_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f127958",
   "metadata": {},
   "source": [
    "# PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6809e3c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# PCA feature importance\n",
    "feature_importance = pd.DataFrame(pca.components_, columns=data_class.data.columns)\n",
    "feature_importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d017687",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(feature_importance.iloc[0], feature_importance.iloc[1], label=feature_importance.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ee57822",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance.T[0].abs().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14891042",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance.T[1].abs().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "854d838a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# centers of mass for each category of records\n",
    "centers = result.pivot_table(values=['x', 'y'], index='mouse', aggfunc='mean')\n",
    "centers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79ef01b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculating cosine similarity\n",
    "features = {'value': [],\n",
    "             'label': [],\n",
    "             'feature': [],\n",
    "             'rating': []\n",
    "            }\n",
    "top_n = 8\n",
    "for i, row in centers.iterrows():\n",
    "    top = feature_importance.apply(lambda col: cosine_similarity([row], [col])[0,0] * (col ** 2).sum() ** (1 / 2))\n",
    "    top = top[top.abs().sort_values(ascending=False).head(top_n).index]\n",
    "    features['value'] += top.tolist()\n",
    "    features['feature'] += top.index.tolist()\n",
    "    features['label'] += [i] * (top_n)\n",
    "    features['rating'] += list(np.arange(top_n))\n",
    "    \n",
    "features = pd.DataFrame(features)\n",
    "features = features.pivot_table(values=['value', 'rating'], index=['label', 'feature'])\n",
    "features = features.sort_values(by=['label', 'rating'])\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0e264b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualization of the most important features by cosine similarity\n",
    "feature_df = features.reset_index()\n",
    "\n",
    "labels = feature_df['label'].unique()\n",
    "\n",
    "fig, axs = plt.subplots(len(labels), 1, figsize=(12, 5*len(labels)))\n",
    "\n",
    "if len(labels) == 1:\n",
    "    axs = [axs]\n",
    "\n",
    "fig.subplots_adjust(hspace=.3)\n",
    "\n",
    "\n",
    "for ax, label in zip(axs, labels):\n",
    "    \n",
    "    sns.barplot(data = feature_df[feature_df['label'] == label], y='feature', x='value', ax=ax)\n",
    "    ax.set_title(label, fontsize=18)\n",
    "    \n",
    "    ax.set_xlabel('value', fontsize=14)\n",
    "    ax.set_ylabel('feature', fontsize=14)\n",
    "    ax.tick_params(axis='both', labelsize=14)\n",
    "    \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "819ddca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_df_button(feature_importance.T.rename(columns={0: 'x', 1: 'y'}), path_to_data+'/feature_weights.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e295e4cc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}