221_ETL_Graph_Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Generacion Features de Grafo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Imports & Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import networkx as nx\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Cargar endpoint\n",
    "infile = open('./data/1_GDB_Layers.pkl','rb')\n",
    "GDB = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "filename = './data/22_Full_Graph.pkl'\n",
    "infile = open(filename, 'rb')\n",
    "MTG = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "filename = './data/22_CTMT_Graph_Dict.pkl'\n",
    "infile = open(filename, 'rb')\n",
    "CTMT_GR = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "filename = './data/22_UNTRD_Conections.pkl'\n",
    "infile = open(filename, 'rb')\n",
    "UNTRD_CONS = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Metricas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['27156187', '27132900', '27137461', ..., '27142620', '27144485',\n",
       "       '27144240'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "UNTRD_CONS.UNTRD_PCON.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "## MEJORAR EFICIENCIA !!!!!!!!!!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['27117493', '80345445', '27117484', '27117498', '80345446', '27117512', '27117496', '27117500', '27117492', '27117510', '27117490', '27117503', '27117515', '27117508', '27117499', '27117514', '27117509', '27117513', '27117511', '27117507', '80345444', '80345447', '27117487', '27117497', '27117495', '27117488', '27117489', '27117486', '27117485', '27117483'])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CTMT_GR.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grafo 27117493 procesado\n",
      "Grafo 80345445 procesado\n",
      "Grafo 27117484 procesado\n",
      "Grafo 27117498 procesado\n",
      "Grafo 80345446 procesado\n",
      "Grafo 27117512 procesado\n",
      "Grafo 27117496 procesado\n",
      "Grafo 27117500 procesado\n",
      "Grafo 27117492 procesado\n",
      "Grafo 27117510 procesado\n",
      "Grafo 27117490 procesado\n",
      "Grafo 27117503 procesado\n",
      "Grafo 27117515 procesado\n",
      "Grafo 27117508 procesado\n",
      "Grafo 27117499 procesado\n",
      "Grafo 27117514 procesado\n",
      "Grafo 27117509 procesado\n",
      "Grafo 27117513 procesado\n",
      "Grafo 27117511 procesado\n",
      "Grafo 27117507 procesado\n",
      "Grafo 80345444 procesado\n",
      "Grafo 80345447 procesado\n",
      "Grafo 27117487 procesado\n",
      "Grafo 27117497 procesado\n",
      "Grafo 27117495 procesado\n",
      "Grafo 27117488 procesado\n",
      "Grafo 27117489 procesado\n",
      "Grafo 27117486 procesado\n",
      "Grafo 27117485 procesado\n",
      "Grafo 27117483 procesado\n"
     ]
    }
   ],
   "source": [
    "untrd_nodes = UNTRD_CONS.UNTRD_PCON.values\n",
    "graph_metrics_df = pd.DataFrame()\n",
    "\n",
    "for G_key in CTMT_GR.keys():\n",
    "    G = CTMT_GR[G_key]\n",
    "    \n",
    "    degree_dict = dict(G.degree(untrd_nodes))\n",
    "    df = pd.DataFrame.from_dict(degree_dict, orient='index', columns=['DEGREE'])\n",
    "    \n",
    "    neig_degree_dict = nx.average_neighbor_degree(G)\n",
    "    df = df.join(pd.DataFrame.from_dict(neig_degree_dict, orient='index', columns=['NEIG_DEGREE']))\n",
    "\n",
    "    betweenness_dict = nx.betweenness_centrality(G)\n",
    "    df = df.join(pd.DataFrame.from_dict(betweenness_dict, orient='index', columns=['BET_CEN']))\n",
    "    \n",
    "    closeness_dict = nx.closeness_centrality(G)\n",
    "    df = df.join(pd.DataFrame.from_dict(closeness_dict, orient='index', columns=['CLO_CEN']))\n",
    "\n",
    "    pagerank_dict = nx.pagerank(G)\n",
    "    df = df.join(pd.DataFrame.from_dict(pagerank_dict, orient='index', columns=['PAGE_RANK']))\n",
    "    \n",
    "    graph_metrics_df = pd.concat([graph_metrics_df, df])\n",
    "    \n",
    "    print(f'Grafo {G_key} procesado')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "graph_metrics_df.index.name = 'UNTRD_PCON'\n",
    "graph_metrics_df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UNTRD_PCON</th>\n",
       "      <th>DEGREE</th>\n",
       "      <th>NEIG_DEGREE</th>\n",
       "      <th>BET_CEN</th>\n",
       "      <th>CLO_CEN</th>\n",
       "      <th>PAGE_RANK</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2488</th>\n",
       "      <td>27140435</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.015211</td>\n",
       "      <td>0.001537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1883</th>\n",
       "      <td>27147080</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.023380</td>\n",
       "      <td>0.005864</td>\n",
       "      <td>0.000774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1666</th>\n",
       "      <td>27131363</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.049962</td>\n",
       "      <td>0.005341</td>\n",
       "      <td>0.000755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3563</th>\n",
       "      <td>27150613</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.006214</td>\n",
       "      <td>0.005147</td>\n",
       "      <td>0.000385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3503</th>\n",
       "      <td>27149407</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0.059657</td>\n",
       "      <td>0.005555</td>\n",
       "      <td>0.000353</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     UNTRD_PCON  DEGREE  NEIG_DEGREE   BET_CEN   CLO_CEN  PAGE_RANK\n",
       "2488   27140435       1          2.0  0.000000  0.015211   0.001537\n",
       "1883   27147080       2          2.5  0.023380  0.005864   0.000774\n",
       "1666   27131363       2          2.0  0.049962  0.005341   0.000755\n",
       "3563   27150613       2          2.5  0.006214  0.005147   0.000385\n",
       "3503   27149407       2          2.5  0.059657  0.005555   0.000353"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph_metrics_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3882, 6)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph_metrics_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3848, 6)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "UNTRD_CONS.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "UNTRD_GraphMetrics = pd.merge(UNTRD_CONS[['COD_ID','UNTRD_PCON']],graph_metrics_df, on='UNTRD_PCON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Retirar las columnas de keys\n",
    "UNTRD_GraphMetrics.drop('UNTRD_PCON', axis=1, inplace= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COD_ID</th>\n",
       "      <th>DEGREE</th>\n",
       "      <th>NEIG_DEGREE</th>\n",
       "      <th>BET_CEN</th>\n",
       "      <th>CLO_CEN</th>\n",
       "      <th>PAGE_RANK</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3257</th>\n",
       "      <td>26876676</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.009257</td>\n",
       "      <td>0.003280</td>\n",
       "      <td>0.000258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>26877139</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.042952</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.000258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2593</th>\n",
       "      <td>26877238</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.086185</td>\n",
       "      <td>0.007870</td>\n",
       "      <td>0.000651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1775</th>\n",
       "      <td>26879783</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.213628</td>\n",
       "      <td>0.045070</td>\n",
       "      <td>0.005489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3521</th>\n",
       "      <td>26878769</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005260</td>\n",
       "      <td>0.000457</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        COD_ID  DEGREE  NEIG_DEGREE   BET_CEN   CLO_CEN  PAGE_RANK\n",
       "3257  26876676       2          2.0  0.009257  0.003280   0.000258\n",
       "333   26877139       2          2.0  0.042952  0.002924   0.000258\n",
       "2593  26877238       2          2.0  0.086185  0.007870   0.000651\n",
       "1775  26879783       2          2.0  0.213628  0.045070   0.005489\n",
       "3521  26878769       1          2.0  0.000000  0.005260   0.000457"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "UNTRD_GraphMetrics.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Exportar exit point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "filename = f'./data/221_Graph_Metrics.pkl'\n",
    "outfile = open(filename, 'wb')\n",
    "pickle.dump(UNTRD_GraphMetrics, outfile)\n",
    "outfile.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}