Appendix_4.2_processing_neighbors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:02:38.154736Z",
     "start_time": "2021-10-14T15:02:36.200788Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import seaborn as sns\n",
    "import dask\n",
    "from dask.diagnostics import ProgressBar\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:02:35.230812Z",
     "start_time": "2021-10-14T15:02:35.227630Z"
    }
   },
   "outputs": [],
   "source": [
    "tmp_data_path = '../MA_data/data/tmp'\n",
    "data_path = '../MA_data/data'\n",
    "\n",
    "s_year = 1997\n",
    "e_year = 2020\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:02:40.764032Z",
     "start_time": "2021-10-14T15:02:40.650601Z"
    }
   },
   "outputs": [],
   "source": [
    "sdc_tnic = pd.read_pickle(tmp_data_path+f\"/sdc_tnic_{s_year}_{e_year}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:02:57.674482Z",
     "start_time": "2021-10-14T15:02:44.463069Z"
    }
   },
   "outputs": [],
   "source": [
    "tnic = pd.read_pickle(tmp_data_path+f\"/tnic2_{s_year-1}_{e_year-1}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:02:57.759944Z",
     "start_time": "2021-10-14T15:02:57.676804Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(tmp_data_path+f\"/tnic_info_3_pairs_{s_year-1}_{e_year-1}\", 'rb') as f:\n",
    "    gvkey_lsts, key_ind_maps , ind_key_maps = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:07:40.609310Z",
     "start_time": "2021-10-14T15:07:40.606478Z"
    }
   },
   "outputs": [],
   "source": [
    "cpu_cores = os.cpu_count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:12:50.727857Z",
     "start_time": "2021-10-14T15:12:50.721270Z"
    }
   },
   "outputs": [],
   "source": [
    "def find_top_peer_lst(tnic, gvkey, year, k, thres):\n",
    "    '''\n",
    "    Obtain top peers \n",
    "    tnic has: year, gvkey1, gvkey2, score, rank\n",
    "    '''\n",
    "    top_peers_subdf =  tnic[(tnic.year == year)]\n",
    "    top_peers_subdf = top_peers_subdf[(top_peers_subdf.gvkey1 == gvkey)] \n",
    "    top_peers = top_peers_subdf[(top_peers_subdf['score'] >= thres)]\n",
    "    \n",
    "    if top_peers.shape[0] < k:\n",
    "        top_peers_10 = top_peers_subdf[(top_peers_subdf['yearly_rank'] <= k)]\n",
    "        if top_peers.shape[0] < top_peers_10.shape[0]:\n",
    "            top_peers = top_peers_10\n",
    "    \n",
    "    top_peers_lst = top_peers.gvkey2.tolist()\n",
    "    return top_peers_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:12:51.008693Z",
     "start_time": "2021-10-14T15:12:51.002672Z"
    }
   },
   "outputs": [],
   "source": [
    "@dask.delayed\n",
    "def create_top_peer_map(tnic, year, gvkey_lsts, k=10, thres=0.4):\n",
    "    y = year\n",
    "    year_lst = {}\n",
    "    #print(\"processing year \", y)\n",
    "    for gvkey1 in gvkey_lsts[y]:\n",
    "        year_lst[gvkey1] = find_top_peer_lst(tnic, gvkey1, y, k, thres)\n",
    "    with open(tmp_data_path+f\"/top_peers_{y}\", 'wb') as f:\n",
    "        pickle.dump(year_lst, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-14T15:12:56.879722Z",
     "start_time": "2021-10-14T15:12:51.316287Z"
    }
   },
   "outputs": [],
   "source": [
    "update_tasks_s = [create_top_peer_map(tnic, year, gvkey_lsts) for year in range(1996, 2019+1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-15T04:08:33.359598Z",
     "start_time": "2021-10-14T15:12:56.881450Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[                                        ] | 0% Completed |  9hr  6min 10.7s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed | 12hr 55min 36.1s\n"
     ]
    }
   ],
   "source": [
    "with ProgressBar():\n",
    "    dask.compute(update_tasks_s, num_workers = cpu_cores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "GNN",
   "language": "python",
   "name": "gnn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}