neo4j.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import SVO\n",
    "import neo4j\n",
    "import re\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from multiprocessing import Process, Queue, Pool \n",
    "from tqdm import tqdm \n",
    "import boto3\n",
    "import os\n",
    "from cathay.config import ApplicationConfig\n",
    "from cathay.util.SimplePool.threadpool import ThreadPool\n",
    "from cathay.util.SimplePool.thread_job import ThreadJob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 讀檔"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"{ApplicationConfig.get_reuters_path()}tesla.pkl\", 'rb') as f:\n",
    "    data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 整理格式\n",
    "result = []\n",
    "data['date'] = data['date'].apply(lambda x:time.strptime(x.split('/')[0], '%B %d, %Y '))\n",
    "data['date'] = data['date'].apply(lambda x:time.strftime('%Y-%m-%d',x))\n",
    "result = data.to_dict('records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_set = list(set([x['date'] for x in result]))\n",
    "date_set.sort(reverse=True)\n",
    "month_set = list(set([x[:7] for x in date_set]))\n",
    "month_set.sort(reverse=True)\n",
    "# month_set.pop(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<BarContainer object of 13 artists>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA20AAAD4CAYAAAB/sQ6nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAXOUlEQVR4nO3df7CldX0f8PcnLP7+BXIlCNhFJTGk06DdoPnR1kgTEaYBW6PYqRKrQ2JlovkxzZrOVNMpHWwTnbHTmMFixNSK1B+RiLEiMVU7CboaRH4EWXWNbBA2Kv4Yqwn46R/n2fFm2WXv3XvP3u/e83rNnDnP+T7Pc77fz5yzZ8/7PM/zvdXdAQAAYEzft9EDAAAA4MCENgAAgIEJbQAAAAMT2gAAAAYmtAEAAAxsy0YPIEmOO+643rp160YPAwAAYEN84hOf+OvuXtrfuiFC29atW7Njx46NHgYAAMCGqKovHGid0yMBAAAGJrQBAAAMTGgDAAAYmNAGAAAwMKENAABgYEIbAADAwIQ2AACAgQltAAAAAztoaKuqB1XVx6rqU1V1U1X95tR+SlVdV1U7q+rtVfWAqf2B0+Od0/qt8y0BAABg89qygm2+k+QZ3f3Nqjo6yUer6o+S/EqS13X3FVX1u0lenOQN0/1Xu/uJVXV+ktcked6cxg8AwCazdfvVG9LvrkvO2ZB+4WAOeqStZ745PTx6unWSZyR5x9R+eZLzpuVzp8eZ1p9ZVbVuIwYAAFggK7qmraqOqqrrk9yV5Jokn01yd3ffM21ye5ITp+UTk3wxSab1X0vy6P0854VVtaOqduzZs2dtVQAAAGxSKwpt3X1vd5+e5KQkZyR50lo77u5Lu3tbd29bWlpa69MBAABsSquaPbK7707yoSQ/luRRVbX3mriTkuyelncnOTlJpvWPTPLldRktAADAglnJ7JFLVfWoafnBSX46yS2ZhbfnTJtdkOQ90/JV0+NM6/+4u3s9Bw0AALAoVjJ75AlJLq+qozILeVd293ur6uYkV1TVf0zy50kum7a/LMnvV9XOJF9Jcv4cxg0AALAQDhrauvuGJE/eT/vnMru+bd/2byf5uXUZHQAAwIJb1TVtAAAAHF5CGwAAwMCENgAAgIEJbQAAAAMT2gAAAAYmtAEAAAxMaAMAABiY0AYAADAwoQ0AAGBgQhsAAMDAhDYAAICBCW0AAAADE9oAAAAGJrQBAAAMTGgDAAAYmNAGAAAwMKENAABgYEIbAADAwIQ2AACAgQltAAAAAxPaAAAABia0AQAADExoAwAAGJjQBgAAMDChDQAAYGAHDW1VdXJVfaiqbq6qm6rq5VP7q6tqd1VdP93OXrbPK6tqZ1XdWlXPnGcBAAAAm9mWFWxzT5Jf7e5PVtXDk3yiqq6Z1r2uu39r+cZVdVqS85P8cJLHJvlgVf1Ad9+7ngMHAABYBAc90tbdd3T3J6flbyS5JcmJ97PLuUmu6O7vdPfnk+xMcsZ6DBYAAGDRrOqatqramuTJSa6bmi6qqhuq6k1VdczUdmKSLy7b7fbsJ+RV1YVVtaOqduzZs2fVAwcAAFgEKw5tVfWwJO9M8oru/nqSNyR5QpLTk9yR5LdX03F3X9rd27p729LS0mp2BQAAWBgrCm1VdXRmge2t3f2uJOnuO7v73u7+bpI35nunQO5OcvKy3U+a2gAAAFillcweWUkuS3JLd792WfsJyzZ7dpIbp+WrkpxfVQ+sqlOSnJrkY+s3ZAAAgMWxktkjfyLJC5J8uqqun9p+I8nzq+r0JJ1kV5JfSJLuvqmqrkxyc2YzT77MzJEAAACH5qChrbs/mqT2s+p997PPxUkuXsO4AAAAyCpnjwQAAODwEtoAAAAGJrQBAAAMTGgDAAAYmNAGAAAwMKENAABgYEIbAADAwIQ2AACAgQltAAAAAxPaAAAABia0AQAADExoAwAAGJjQBgAAMDChDQAAYGBbNnoAAIfD1u1Xb0i/uy45Z0P6BQA2D0faAAAABia0AQAADExoAwAAGJjQBgAAMDChDQAAYGBCGwAAwMCENgAAgIEJbQAAAAMT2gAAAAYmtAEAAAzsoKGtqk6uqg9V1c1VdVNVvXxqP7aqrqmq26b7Y6b2qqrXV9XOqrqhqp4y7yIAAAA2qy0r2OaeJL/a3Z+sqocn+URVXZPk55Nc292XVNX2JNuT/HqSZyU5dbo9NckbpnsAAGA/tm6/ekP63XXJORvSL6tz0CNt3X1Hd39yWv5GkluSnJjk3CSXT5tdnuS8afncJG/pmT9L8qiqOmHdRw4AALAAVnVNW1VtTfLkJNclOb6775hWfSnJ8dPyiUm+uGy326e2fZ/rwqraUVU79uzZs8phAwAALIYVh7aqeliSdyZ5RXd/ffm67u4kvZqOu/vS7t7W3duWlpZWsysAAMDCWFFoq6qjMwtsb+3ud03Nd+497XG6v2tq353k5GW7nzS1AQAAsEormT2yklyW5Jbufu2yVVcluWBaviDJe5a1v3CaRfJpSb627DRKAAAAVmEls0f+RJIXJPl0VV0/tf1GkkuSXFlVL07yhSTPnda9L8nZSXYm+VaSF63riAEAABbIQUNbd380SR1g9Zn72b6TvGyN4wIAACCrnD0SAACAw0toAwAAGJjQBgAAMLCVTEQCwBFo6/arN6TfXZecsyH9AsBm5UgbAADAwIQ2AACAgQltAAAAAxPaAAAABia0AQAADExoAwAAGJjQBgAAMDB/pw0AjlAb9bf4En+PD+BwcqQNAABgYEIbAADAwIQ2AACAgQltAAAAAxPaAAAABmb2SFhAGzXjnNnmAABWz5E2AACAgQltAAAAAxPaAAAABia0AQAADExoAwAAGJjQBgAAMDChDQAAYGAHDW1V9aaququqblzW9uqq2l1V10+3s5ete2VV7ayqW6vqmfMaOAAAwCJYyZG2Nyc5az/tr+vu06fb+5Kkqk5Lcn6SH572+Z2qOmq9BgsAALBoDhrauvvDSb6ywuc7N8kV3f2d7v58kp1JzljD+AAAABbaWq5pu6iqbphOnzxmajsxyReXbXP71HYfVXVhVe2oqh179uxZwzAAAAA2r0MNbW9I8oQkpye5I8lvr/YJuvvS7t7W3duWlpYOcRgAAACb2yGFtu6+s7vv7e7vJnljvncK5O4kJy/b9KSpDQAAgENwSKGtqk5Y9vDZSfbOLHlVkvOr6oFVdUqSU5N8bG1DBAAAWFxbDrZBVb0tydOTHFdVtyd5VZKnV9XpSTrJriS/kCTdfVNVXZnk5iT3JHlZd987n6EDAGx+W7dfvWF977rknA3rG/ieg4a27n7+fpovu5/tL05y8VoGBQAAwMxaZo8EAABgzoQ2AACAgQltAAAAAxPaAAAABia0AQAADExoAwAAGJjQBgAAMDChDQAAYGBCGwAAwMCENgAAgIEJbQAAAAMT2gAAAAYmtAEAAAxsy0YPAADWw9btV29Y37suOWfD+gZg83OkDQAAYGBCGwAAwMCENgAAgIEJbQAAAAMT2gAAAAYmtAEAAAxMaAMAABiY0AYAADAwf1wbADhi+CPqwCJypA0AAGBgQhsAAMDADhraqupNVXVXVd24rO3Yqrqmqm6b7o+Z2quqXl9VO6vqhqp6yjwHDwAAsNmt5Ejbm5OctU/b9iTXdvepSa6dHifJs5KcOt0uTPKG9RkmAADAYjpoaOvuDyf5yj7N5ya5fFq+PMl5y9rf0jN/luRRVXXCeg0WAABg0RzqNW3Hd/cd0/KXkhw/LZ+Y5IvLtrt9aruPqrqwqnZU1Y49e/Yc4jAAAAA2tzVPRNLdnaQPYb9Lu3tbd29bWlpa6zAAAAA2pUMNbXfuPe1xur9rat+d5ORl2500tQEAAHAIDjW0XZXkgmn5giTvWdb+wmkWyacl+dqy0ygBAABYpS0H26Cq3pbk6UmOq6rbk7wqySVJrqyqFyf5QpLnTpu/L8nZSXYm+VaSF81hzMzR1u1Xb1jfuy45Z8P6BgCAUR00tHX38w+w6sz9bNtJXrbWQQEAADCz5olIAAAAmB+hDQAAYGAHPT0SNruNuo7PNXwAAKyEI20AAAADE9oAAAAGJrQBAAAMTGgDAAAYmNAGAAAwMKENAABgYEIbAADAwIQ2AACAgQltAAAAAxPaAAAABia0AQAADGzLRg9gZFu3X71hfe+65JwN6xsAAOZto75rH4nfsx1pAwAAGJjQBgAAMDChDQAAYGBCGwAAwMCENgAAgIEJbQAAAAMT2gAAAAYmtAEAAAxMaAMAABiY0AYAADCwLWvZuap2JflGknuT3NPd26rq2CRvT7I1ya4kz+3ur65tmAAAAItpPY60/VR3n97d26bH25Nc292nJrl2egwAAMAhmMfpkecmuXxavjzJeXPoAwAAYCGsNbR1kg9U1Seq6sKp7fjuvmNa/lKS49fYBwAAwMJa0zVtSX6yu3dX1WOSXFNVf7F8ZXd3VfX+dpxC3oVJ8rjHPW6NwwAAANic1nSkrbt3T/d3JXl3kjOS3FlVJyTJdH/XAfa9tLu3dfe2paWltQwDAABg0zrk0FZVD62qh+9dTvIzSW5MclWSC6bNLkjynrUOEgAAYFGt5fTI45O8u6r2Ps//7O73V9XHk1xZVS9O8oUkz137MAEAABbTIYe27v5ckh/ZT/uXk5y5lkEBAAAwM48p/wEAAFgnQhsAAMDAhDYAAICBCW0AAAADE9oAAAAGJrQBAAAMTGgDAAAYmNAGAAAwMKENAABgYEIbAADAwIQ2AACAgQltAAAAAxPaAAAABia0AQAADExoAwAAGJjQBgAAMDChDQAAYGBCGwAAwMCENgAAgIEJbQAAAAMT2gAAAAYmtAEAAAxMaAMAABiY0AYAADAwoQ0AAGBgQhsAAMDA5hbaquqsqrq1qnZW1fZ59QMAALCZzSW0VdVRSf5bkmclOS3J86vqtHn0BQAAsJnN60jbGUl2dvfnuvtvklyR5Nw59QUAALBpVXev/5NWPSfJWd39kunxC5I8tbsvWrbNhUkunB7+YJJb130gG+u4JH+90YM4zNS8GNS8GNS8GBat5kWrN1HzolDz5vD3untpfyu2HO6R7NXdlya5dKP6n7eq2tHd2zZ6HIeTmheDmheDmhfDotW8aPUmal4Uat785nV65O4kJy97fNLUBgAAwCrMK7R9PMmpVXVKVT0gyflJrppTXwAAAJvWXE6P7O57quqiJP87yVFJ3tTdN82jr4Ft2lM/74eaF4OaF4OaF8Oi1bxo9SZqXhRq3uTmMhEJAAAA62Nuf1wbAACAtRPaAAAABia0Tarq5Kr6UFXdXFU3VdXLp/Zjq+qaqrptuj9man9SVf1pVX2nqn5tn+d6eVXdOD3PK+6nz7Oq6taq2llV25e1n1lVn6yq66vqo1X1xAWo+SNTvddX1V9V1R8sQM1VVRdX1Weq6paq+qUFqPnNVfX5Za/16Zu95mXrX19V31zvWpc9/zA1V9VlVfWpqrqhqt5RVQ9bgJrfOrXfWFVvqqqjF6Dmi6a2rqrjjoB6f3l6jhur6m1V9aAD9Pn+qrq7qt67T/uR+Bqvtea5v8YD1nxKVV031f32mk2ot1lqvmB63tuq6oJl7c+r2ef1TVX1mnnUO2DNz6+qT091v3+e7+91091us+v6TkjylGn54Uk+k+S0JP85yfapfXuS10zLj0nyo0kuTvJry57n7ye5MclDMpvo5YNJnrif/o5K8tkkj0/ygCSfSnLatO4zSX5oWv43Sd682WveZ7t3JnnhZq85yYuSvCXJ9+3tawFqfnOS58yjzlFrntZvS/L7Sb65CDUnecSy7V67t/9NXvPZSWq6vS3JSxeg5icn2ZpkV5LjBq/3xCSfT/Lg6fGVSX7+AH2emeSfJXnvPu1H2mu8HjXP/TUesOYrk5w/Lf/uZnmdkxyb5HPT/THT8jFJHp3kL5MsTdtdnuTMTV7zliR37X1PT/2/el7v7/W6OdI26e47uvuT0/I3ktyS2Zvi3MzewJnuz5u2uau7P57kb/d5qh9Kcl13f6u770nyf5L88/10eUaSnd39ue7+myRXTH0lSSd5xLT8yCR/tQ4l3sdgNSdJquoRSZ6RZC5H2gar+aVJ/kN3f3dvX+tU5t8xWM2HxUg1V9VRSf5Lkn+7jiXex0g1d/fXk9nR5CQPzuwzbd0NVvP7epLkY5n9fdJ1N1jNf97du9azvn2tY73J7Ivag6tqS2Zhdb//t3b3tUm+sZ/2I+01TtZe89xf46mfIWqePrOekeQd+/a53jag5mcmuaa7v9LdX01yTZKzMvtB5rbu3jNt98Ek/2IdSryPgWre++PLQ6fX/BEH2H8oQtt+VNXWzH5dui7J8d19x7TqS0mOP8juNyb5R1X16Kp6SGa/zJ28n+1OTPLFZY9vn9qS5CVJ3ldVtyd5QZJLDqGMVRmg5r3OS3Lt3i998zRAzU9I8ryq2lFVf1RVpx5SIaswQM1JcvF0OsLrquqBq69idQao+aIkVy3rd+4GqDlV9XtTf09K8l9XX8XqjFDzNI6jM/vcfv8qS1i1UWo+XNZSb3fvTvJbmR1RuCPJ17r7A4c4jiPiNV7Pmg+nDa750Ununn7ISA7T+/0w1Xygf8s7k/xgVW2dAtB52f9nwbrayJq7+28z++H805mFtdOSXHaotRwuQts+anbtxTuTvGLf4DD9una/vxh39y1JXpPkA5l9oF+f5N5VDuOXk5zd3Scl+b3MTi+am0Fq3uv5mZ12MleD1PzAJN/u7m1J3pjkTavcf1UGqfmVmX2J/9HMTlf49VXuvyobXXNVPTbJz+UwhJZlfY7wOqe7X5TksZn9kvq81e6/GqPUPPmdJB/u7o8c4v4rMljNc7fWeqdrZM5Nckpm78uHVtW/OsThHBGv8TrXfFio+fDXPB2BemmStyf5SGanws71s2Cja55+eHlpZqHxsUluyOz7ydCEtmWmF/GdSd7a3e+amu+sqhOm9Sdkdg7s/eruy7r7H3b3P07y1SSfmS6+3Dv5wi8m2Z2/+0vGSUl2V9VSkh/p7uum9rcn+fF1KXA/Rqh52ViOy+xUnKvXo7YDGajm25Ps7f/dSf7BWms7kFFqnk6N6O7+TmY/SJyxXjXua5Can5zkiUl2VtWuJA+pqp3rVOJ9DFLz8ue5N7PT6eZyqk0yVs1V9aokS0l+ZT1qO5CRaj4c1qnef5rk8929Z/qV/V1Jfryqnrqs3p9dwViOpNd4XWo+XAap+ctJHjUdcUrm/H4/zDXf3//Nf9jdT+3uH0tya2bXms3FIDWfniTd/dkpJF6ZOX7XXi9bDr7JYqiqyuzQ6C3dvfzI1lVJLsjsFMULkrxnBc/1mO6+q6oel9k1Ak/r7rszvUmmbbYkObWqTsnsDXR+kn+Z2X+cj6yqH+juzyT56cx+qV53A9W813MyuyD422ur7H7HOVLNf5DkpzK7mPafZE4fkiPVXFUndPcd05jOy+wUrXU3Ss3dfVOS71+23Te7e16zwQ5R8zSOJ3T3zmn5Z5P8xboUed9xDlHztO4lmV0/cWZP16nOw0g1Hw7rWO9fJnlazU4F/X+ZTUKxY/qBdEWz2B6Br/Gaaz5cRqm5u7uqPpTZ95ErVtjnITncNVfVsUn+U00zMyb5mUxHl5Z9FhyT2QR4z11rffszUM0PSnJaVS317Fq+uX3XXlc9wGwoI9yS/GRmh2NvyOw0keszO8f/0UmuTXJbZhdnHjtt//2ZHSn5epK7p+VHTOs+kuTmzGbZOuAMPNPzfyazmbn+3bL2Z2d2nu2nkvxJksdv9pqndX+S5KwFep0fldlRxU8n+dPMjrBu9pr/eKr3xiT/I8nDNnvN+2wzz9kjh6g5szM4/u+y1/mtWTab5GaseWq/Z2rbO45/vwA1/9L0fPdkdl3Ifx+83t/M7AeEGzObzfWBB+jzI0n2ZPZl8PYkzzyCX+O11jz313jAmh+f2UQzO5P8rwPtf4TW/K+nunYmedGy9rdl9llwc6aZMxeg5l/MLKjdkOQPkzx6XnWv162mgQMAADAg17QBAAAMTGgDAAAYmNAGAAAwMKENAABgYEIbAADAwIQ2AACAgQltAAAAA/v/3pGCyhBqREkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num = []\n",
    "for i in month_set:\n",
    "    num.append(len([x for x in result if i in x['date']]))\n",
    "\n",
    "plt.figure(figsize=(15,4))\n",
    "plt.bar(month_set, num, width = 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1960"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find SVO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "258"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(date_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reuters_Tesla_1 has already existed.\n"
     ]
    }
   ],
   "source": [
    "graph = neo4j.Neo4j_File()\n",
    "folder = 'Reuters_Tesla_1'\n",
    "# folder = 'Reuters_Huawei'\n",
    "graph.Create_Folder(folder, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do(sent, date, new, count, tag):  \n",
    "#     try:     \n",
    "        sent = re.sub(\"[+\\!\\/_$%^*()-+.:\\\"\\'“”]+|[+——！，。？、~@#￥%……&*（）：`]+\", '', sent)\n",
    "        sent = sent.replace('Headlines', '')\n",
    "        svo = SVO.SVO(sent)\n",
    "        svo_result = svo.find_svo()\n",
    "        if svo_result != 'Sentence can not find SVO.':\n",
    "            graph.ToFile(svo_result, sent, date, new, count, tag)\n",
    "        \n",
    "#     except Exception as e:\n",
    "#             print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 258/258 [00:00<00:00, 276.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103.40716028213501\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    news_id = []\n",
    "    sent_pool = []\n",
    "    t = ThreadPool(12)\n",
    "    start = time.time()\n",
    "    source = 'Reuters'\n",
    "    for date in tqdm(range(len(date_set))):\n",
    "        sentence = [x for x in result if x['date'] == date_set[date]]\n",
    "\n",
    "        for i in sentence:\n",
    "            new = hash(i['title']+i['date'])\n",
    "            if new not in [x['id'] for x in news_id]:\n",
    "                news_id.append({'id':new, 'title':i['title']+i['date'], 'text':i['text']})\n",
    "                sent = [s for s in sent_tokenize(i['text']) if 'Tesla' in s]\n",
    "                sent = [s for s in sent if s not in sent_pool]\n",
    "                sent_pool += sent\n",
    "                count = 0\n",
    "                \n",
    "                for j in range(len(sent)): \n",
    "                    if j == 0 and '-' in sent[j]:\n",
    "                        sent[j] = sent[j][sent[j].index('-')+1:]\n",
    "\n",
    "                    job = ThreadJob(do, (sent[j], i['date'], new, count, source))\n",
    "                    t.add_job(job)\n",
    "                    count += 1\n",
    "\n",
    "    t.start()\n",
    "    t.finish()  \n",
    "    end = time.time()\n",
    "    print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(ApplicationConfig.get_neo4j_svo_output_path()+folder+'/news_id.pkl', 'wb') as f:\n",
    "    pickle.dump(news_id, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# News' Entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def name_entity(sent, Id, path, comprehend):\n",
    "    entity = []\n",
    "    \n",
    "    for text in sent:\n",
    "        entity += comprehend.detect_entities(Text=text, LanguageCode='en')['Entities']\n",
    "        \n",
    "    with open(path+'/'+str(Id)+'.pkl', 'wb') as f:\n",
    "        pickle.dump(entity, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1050/1050 [00:00<00:00, 1479.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "221.61590003967285\n"
     ]
    }
   ],
   "source": [
    "# name_entity\n",
    "path = ApplicationConfig.get_neo4j_svo_output_path()+folder+'/name_entity'\n",
    "if not os.path.isdir(path):\n",
    "    os.mkdir(path)\n",
    "    print('Create Successfully')\n",
    "    \n",
    "aws_nlu_config =  ApplicationConfig.get_aws_nlu_config()\n",
    "comprehend = boto3.client(aws_access_key_id=aws_nlu_config['access_key'], aws_secret_access_key=aws_nlu_config['secret_key'], service_name='comprehend', region_name=aws_nlu_config['region'])\n",
    "t = ThreadPool(4)\n",
    "start = time.time()\n",
    "for i in tqdm(news_id):\n",
    "    sent = [s for s in sent_tokenize(i['text']) if 'Tesla' in s]\n",
    "#     name_entity(sent, i['id'], path, comprehend)\n",
    "    job = ThreadJob(name_entity, (sent, i['id'], path, comprehend))\n",
    "    t.add_job(job)\n",
    "    \n",
    "t.start()\n",
    "t.finish()      \n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/s3/Financial_News_Extraction/src/cathay/re/neo4j.py:312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tmp['token'] = tmp.apply(lambda x: nltk.word_tokenize(x['start']+x['end']), axis = 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training word2vec model...\n",
      "Finished!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<neo4j.Save at 0x7f4d280e4048>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 資料夾內的Node Integrate\n",
    "# Add Entity\n",
    "folder = 'Reuters_Tesla_1'\n",
    "neo4j.Save(folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 匯入Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "load = neo4j.Csv2Neo4j()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "folder = 'Reuters_Tesla'\n",
    "# import Node\n",
    "load.Load(folder)\n",
    "# import Relation\n",
    "load.Load(folder, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SNA指標"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "graph = neo4j.SNA_Indicator(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "degree_centrality = graph.Centrality('Degree')\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'Degree Centrality')\n",
    "\n",
    "degree_centrality = graph.Centrality('Degree', False)\n",
    "degree_centrality = degree_centrality.sort_values('2019-07-31', ascending = False)\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'Degree Centrality')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "degree_centrality = graph.Centrality('Betweeness')\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'Betweeness Centrality')\n",
    "\n",
    "degree_centrality = graph.Centrality('Betweeness', False)\n",
    "degree_centrality = degree_centrality.sort_values('2019-07-31', ascending = False)\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'Betweeness Centrality')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "degree_centrality = graph.Centrality('Closeness')\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'Closeness Centrality')\n",
    "\n",
    "degree_centrality = graph.Centrality('Closeness', False)\n",
    "degree_centrality = degree_centrality.sort_values('2019-07-31', ascending = False)\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'Closeness Centrality')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "degree_centrality = graph.Centrality('PageRank')\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'PageRank Centrality')\n",
    "\n",
    "degree_centrality = graph.Centrality('PageRank', False)\n",
    "degree_centrality = degree_centrality.sort_values('2019-07-31', ascending = False)\n",
    "graph.Plot(degree_centrality.iloc[1:6], 'PageRank Centrality')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph.Plot(degree_centrality.iloc[1:10], 'PageRank Centrality')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "degree_centrality.sort_values('2019-07-31', ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stock Price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "price = graph.Load_data('dataset/us_stock_prices/FinancialData/TSLA/TSLA_history_prices_daily.csv', '2019-06-01')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "graph.Plot(price, 'Stock Price')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create NetworkX from csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cathay.config import ApplicationConfig\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "path = ApplicationConfig.get_neo4j_svo_output_path() + 'Reuters_Tesla'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "rel = pd.read_csv(path+'/Relation_revise.csv')\n",
    "Graphtype = nx.Graph()\n",
    "G = nx.from_pandas_edgelist(rel, 'start', 'end', create_using = Graphtype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Node2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing transition probabilities: 100%|██████████| 9882/9882 [00:11<00:00, 895.29it/s] \n"
     ]
    }
   ],
   "source": [
    "from node2vec import Node2Vec\n",
    "node2vec = Node2Vec(G, dimensions=128, walk_length=3, num_walks=200, workers=4)\n",
    "model = node2vec.fit(window=3, min_count=1, batch_words=4)\n",
    "# model.save('node2vec_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-aca7b7f21fc4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmost_similar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'tesla'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "model.most_similar('tesla')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.most_similar('10') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('node2vec_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from py2neo import Node, Relationship, Graph\n",
    "graph = Graph(\"http://13.112.207.201:7474/browser\", user = 'neo4j', password = 'neo4j@cathay')\n",
    "graph.delete_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}