Appendix_3.3_TNIC_top_k_peers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:55:55.459353Z",
     "start_time": "2021-10-12T23:55:55.457204Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "from os.path import join as pjoin\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:55:55.483244Z",
     "start_time": "2021-10-12T23:55:55.460742Z"
    }
   },
   "outputs": [],
   "source": [
    "tmp_data_path = '../MA_data/data/tmp/'\n",
    "data_path = '../MA_data/data/'\n",
    "\n",
    "s_year = 1996\n",
    "e_year = 2019\n",
    "k = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:55:55.490314Z",
     "start_time": "2021-10-12T23:55:55.484907Z"
    }
   },
   "outputs": [],
   "source": [
    "def load_tnic(data_path, s_year, e_year):\n",
    "    tnic3 =  pd.read_csv(data_path+'tnic3_data/tnic3_data.txt', delimiter = '\\t' )\n",
    "    tnic = tnic3[(tnic3['year'] <= e_year) & (tnic3['year'] >= s_year)]\n",
    "\n",
    "\n",
    "    tnic['gvkey1'] = tnic['gvkey1'].astype(str)\n",
    "    tnic['gvkey2'] = tnic['gvkey2'].astype(str)\n",
    "\n",
    "    tnic = tnic.dropna(subset=['score'])\n",
    "    return tnic\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:55:55.501593Z",
     "start_time": "2021-10-12T23:55:55.492213Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_gvkeylst_adj(tnic, year, top_peer=10, get_adj=False, weighted=False):\n",
    "\n",
    "    def add_rank(df):\n",
    "        df['rank'] = df.groupby('gvkey1')['score'].rank('dense', ascending=False)\n",
    "        df['rank'] = df['rank'].astype(int)\n",
    "        return df\n",
    "    \n",
    "    tnic_year = tnic[tnic.year == year]\n",
    "\n",
    "    tnic_year = add_rank(tnic_year)\n",
    "    tnic_year_top = tnic_year[tnic_year['rank'] <= top_peer].reset_index(drop=True)\n",
    "    \n",
    "    \n",
    "    gvkey_arr = np.unique(np.concatenate((tnic_year_top.gvkey1.unique(), tnic_year_top.gvkey2.unique()), axis = 0))\n",
    "    gvkey_lst = list(gvkey_arr)\n",
    "    N = len(gvkey_lst)\n",
    "    \n",
    "    gvkey_index_mapping = {value:index for index, value in enumerate(gvkey_lst)}\n",
    "    index_gvkey_mapping = {index:value for index, value in enumerate(gvkey_lst)}\n",
    "    #return tnic_year_top, gvkey_arr gvkey_index_mapping, index_gvkey_mapping\n",
    "    if get_adj:\n",
    "        adj = np.zeros((N,N))\n",
    "        \n",
    "        for _, row in tnic_year_top.iterrows():\n",
    "            \n",
    "            gvkey1 = row['gvkey1']\n",
    "            gvkey2 = row['gvkey2']\n",
    "            score = row['score']\n",
    "            \n",
    "            index1 = gvkey_index_mapping[gvkey1]\n",
    "            index2 = gvkey_index_mapping[gvkey2]\n",
    "#            print(index1, index2)\n",
    "            \n",
    "            \n",
    "            \n",
    "            if weighted:\n",
    "                adj[index1, index2] = score\n",
    "            else:\n",
    "                adj[index1, index2] = 1\n",
    "            \n",
    "        \n",
    "        return gvkey_lst, gvkey_index_mapping, index_gvkey_mapping,   adj    \n",
    "    else:\n",
    "        return gvkey_lst, gvkey_index_mapping, index_gvkey_mapping\n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:55:55.511226Z",
     "start_time": "2021-10-12T23:55:55.503489Z"
    }
   },
   "outputs": [],
   "source": [
    "def sdc_in_tnic(df, gvkey_lst):\n",
    "    '''\n",
    "    df: yearly sdc_df\n",
    "    gvkey_lst: yearly\n",
    "    \n",
    "    return integer 1 , 0 !! not str\n",
    "    '''\n",
    "    \n",
    "    # totally 4 conditions are analysable\n",
    "    def mark1(row):\n",
    "        if (row.AS_GVKEY in gvkey_lst) & (row.TS_GVKEY in gvkey_lst):\n",
    "                return 1\n",
    "        else:\n",
    "            return 0\n",
    "    def mark2(row):\n",
    "        if (row.AS_GVKEY in gvkey_lst) & (row.TP_GVKEY in gvkey_lst):\n",
    "                return 1\n",
    "        else:\n",
    "            return 0\n",
    "    def mark3(row):\n",
    "        if (row.AP_GVKEY in gvkey_lst) & (row.TS_GVKEY in gvkey_lst):\n",
    "                return 1\n",
    "        else:\n",
    "            return 0\n",
    "    def mark4(row):\n",
    "        if (row.AP_GVKEY in gvkey_lst) & (row.TP_GVKEY in gvkey_lst):\n",
    "                return 1\n",
    "        else:\n",
    "            return 0\n",
    "    def mark_all(row):\n",
    "        if (row['AS_TS_TNIC'] == 1) | (row['AS_TP_TNIC'] == 1) | (row['AP_TS_TNIC'] == 1) | (row['AP_TP_TNIC'] == 1) :\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "    \n",
    "    df['AS_TS_TNIC'] = df.apply(mark1, axis=1)\n",
    "    df['AS_TP_TNIC'] = df.apply(mark2, axis=1)\n",
    "    df['AP_TS_TNIC'] = df.apply(mark3, axis=1)\n",
    "    df['AP_TP_TNIC'] = df.apply(mark4, axis=1)\n",
    "    \n",
    "    df['A_T_TNIC'] = df.apply(mark_all, axis=1) # analysable\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:55:55.520275Z",
     "start_time": "2021-10-12T23:55:55.512916Z"
    }
   },
   "outputs": [],
   "source": [
    "def check_coverage(sdc_df, year, k):\n",
    "    '''\n",
    "    check portion of deals matched with GVKEY, could be found in graph\n",
    "    \n",
    "    '''\n",
    "    sdc_df_year = sdc_df[sdc_df.YEAR == year]\n",
    "    gvkey_year, _, _ = get_gvkeylst_adj(tnic, year=year, top_peer=k, get_adj=False)\n",
    "    sdc_df_year2 = sdc_in_tnic(sdc_df_year, gvkey_year)\n",
    "    print(f\"total number of deals with GVKEY match is: {sdc_df_year2.shape[0]}\")\n",
    "    answer = sdc_df_year2.A_T_TNIC.value_counts().values[0]/ sdc_df_year2.shape[0]\n",
    "    print(f\"coverage in year {year} is {str(round(answer, 3))}\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:55:55.578479Z",
     "start_time": "2021-10-12T23:55:55.521713Z"
    }
   },
   "outputs": [],
   "source": [
    "sdc_df = pd.read_pickle(tmp_data_path+'/merged_analysable.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:56:15.556662Z",
     "start_time": "2021-10-12T23:55:55.579883Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-105-ec7000184eb7>:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tnic['gvkey1'] = tnic['gvkey1'].astype(str)\n",
      "<ipython-input-105-ec7000184eb7>:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tnic['gvkey2'] = tnic['gvkey2'].astype(str)\n"
     ]
    }
   ],
   "source": [
    "tnic = load_tnic(data_path, s_year, e_year)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What is the portion of SDC deals can be found in the TNIC graph\n",
    "- around 50% - 60%\n",
    "- roughly totally 12000*0.55 = 6600 deals \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:56:30.445050Z",
     "start_time": "2021-10-12T23:56:15.557840Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of deals with GVKEY match is: 715\n",
      "coverage in year 1996 is 0.648\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1263\n",
      "coverage in year 1998 is 0.508\n",
      "total number of deals with GVKEY match is: 1253\n",
      "coverage in year 1999 is 0.515\n",
      "total number of deals with GVKEY match is: 1122\n",
      "coverage in year 2000 is 0.587\n",
      "total number of deals with GVKEY match is: 940\n",
      "coverage in year 2001 is 0.576\n",
      "total number of deals with GVKEY match is: 762\n",
      "coverage in year 2002 is 0.538\n",
      "total number of deals with GVKEY match is: 767\n",
      "coverage in year 2003 is 0.574\n",
      "total number of deals with GVKEY match is: 748\n",
      "coverage in year 2004 is 0.564\n",
      "total number of deals with GVKEY match is: 717\n",
      "coverage in year 2005 is 0.533\n",
      "total number of deals with GVKEY match is: 682\n",
      "coverage in year 2006 is 0.585\n",
      "total number of deals with GVKEY match is: 673\n",
      "coverage in year 2007 is 0.557\n",
      "total number of deals with GVKEY match is: 493\n",
      "coverage in year 2008 is 0.576\n",
      "total number of deals with GVKEY match is: 495\n",
      "coverage in year 2009 is 0.574\n",
      "total number of deals with GVKEY match is: 507\n",
      "coverage in year 2010 is 0.619\n",
      "total number of deals with GVKEY match is: 450\n",
      "coverage in year 2011 is 0.542\n",
      "total number of deals with GVKEY match is: 493\n",
      "coverage in year 2012 is 0.505\n",
      "total number of deals with GVKEY match is: 432\n",
      "coverage in year 2013 is 0.519\n",
      "total number of deals with GVKEY match is: 456\n",
      "coverage in year 2014 is 0.524\n",
      "total number of deals with GVKEY match is: 442\n",
      "coverage in year 2015 is 0.554\n",
      "total number of deals with GVKEY match is: 381\n",
      "coverage in year 2016 is 0.541\n",
      "total number of deals with GVKEY match is: 356\n",
      "coverage in year 2017 is 0.607\n",
      "total number of deals with GVKEY match is: 302\n",
      "coverage in year 2018 is 0.556\n",
      "total number of deals with GVKEY match is: 266\n",
      "coverage in year 2019 is 0.534\n"
     ]
    }
   ],
   "source": [
    "\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    for year in range(s_year, e_year+1, 1):\n",
    "        check_coverage(sdc_df, year, k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# If increase k of top-k peers could increase coverage?\n",
    "- No, but a single event can update more firms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Link with SDC\n",
    "\n",
    "How many SDC deals could be found in the graph (Both side could be found in this list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only the following 4 conditons are analysable:\n",
    "\n",
    "| AP | AS | TS | TP |\n",
    "|----|----|----|----|\n",
    "|    | 1  | 1  |    |\n",
    "|    | 1  |    | 1  |\n",
    "| 1  |    | 1  |    |\n",
    "| 1  |    |    | 1  |\n",
    "\n",
    "- the blank place could be either 0 or 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-12T23:56:43.289658Z",
     "start_time": "2021-10-12T23:56:30.446095Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n",
      "total number of deals with GVKEY match is: 1176\n",
      "coverage in year 1997 is 0.509\n"
     ]
    }
   ],
   "source": [
    "year = 1997\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "    for k in range(10, 100, 10):\n",
    "        check_coverage(sdc_df, year, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "GNN",
   "language": "python",
   "name": "gnn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}