EconomicReportSentiment.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:ea8df4690d7edad87e7228fe83d4569de064400dcab9e504db5242cbd1cf80bf"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from datetime import datetime as dt\n",
      "import numpy as np\n",
      "import bs4 as Soup\n",
      "import requests\n",
      "import pandas as pd"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "now = dt.now()\n",
      "Year,WeekNum,DOW = now.isocalendar()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "print WeekNum"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "29\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "histEconReports = []\n",
      "for eWeek in np.arange(1,WeekNum+1):\n",
      "    if eWeek < 10:\n",
      "        eWeek = '0' + str(eWeek)\n",
      "    histEconReports.append('http://biz.yahoo.com/c/ec/2014' + str(eWeek) + '.html') "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 137
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "# res = requests.get(histEconReports)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 140
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print histEconReports[1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "http://biz.yahoo.com/c/ec/201402.html\n"
       ]
      }
     ],
     "prompt_number": 141
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "def cleanText(text):\n",
      "#     print text\n",
      "    if text == '-' or text == '--':\n",
      "        return ''\n",
      "    cleaned = ''\n",
      "    dash = False\n",
      "    if '-' in text:\n",
      "        dash = True\n",
      "    try:\n",
      "        cleaned = re.search(\"[0-9.]+\",text).group(0)\n",
      "    except:\n",
      "        cleaned = re.search(\"[0-9.]*\",text).group(0)\n",
      "    if dash:\n",
      "        cleaned = \"-\" + cleaned\n",
      "\n",
      "    return cleaned\n",
      "    \n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 146
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import json\n",
      "\n",
      "yahoo_data = []\n",
      "for eHistEconReports in histEconReports[:3]:\n",
      "    res = requests.get(eHistEconReports)\n",
      "    print\n",
      "    print eHistEconReports\n",
      "    soup = Soup.BeautifulSoup(res.content)\n",
      "    table = soup.findAll('table') \n",
      "\n",
      "    trRows = table[5].findAll('tr')\n",
      "\n",
      "    for i, eRow in enumerate(trRows):\n",
      "        listDF = []\n",
      "        if i == 0:  # skip the header row\n",
      "            continue\n",
      "        cols = eRow.findAll('td')        \n",
      "        for eString in np.arange(0,3):\n",
      "            listDF.append(cols[eString].text)\n",
      "        # for columns 3 - 8 clean data\n",
      "        for sIdx in np.arange(4,9):\n",
      "            listDF.append(cleanText(cols[sIdx].string))\n",
      "        yahoo_data.append(listDF)\n",
      "\n",
      "    df = pd.DataFrame(yahoo_data, columns=['Date','Time_ET','Statistic','ForActual','Briefing_Forecast','Market_Expects','Prior','Revised_From'])\n",
      "    #df = pd.DataFrame(yahoo_data)\n",
      "    df = df.fillna(0)\n",
      "    df = df.where(df != '',0)\n",
      "# df['label'] = np.where(df.ForActual > (df.Briefing_Forecast+df.Market_Expects)/2, 1, 0)\n",
      "\n",
      "\n",
      "\n",
      "# df_pop1 = df.pop('Date')\n",
      "# df_pop2 = df.pop('Time(ET)')\n",
      "# df_pop3 = df.pop('Statistic')\n",
      "\n",
      "df[df['Date'] == 'Jan 14']\n",
      "# df.ForActual.ix[3]\n",
      "# df_tmp = df.astype(float)\n",
      "# df= df_pop + df_tmp\n",
      "df.dtypes\n",
      "tofloat = [ u'ForActual', u'Briefing_Forecast', u'Market_Expects', u'Prior', u'Revised_From']\n",
      "df[tofloat] = df[tofloat].astype(float)\n",
      "df[df['Date'] == 'Jan 14']\n",
      "df.dtypes"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "http://biz.yahoo.com/c/ec/201401.html\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "http://biz.yahoo.com/c/ec/201402.html\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "http://biz.yahoo.com/c/ec/201403.html\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 163,
       "text": [
        "Date                  object\n",
        "Time_ET               object\n",
        "Statistic             object\n",
        "ForActual            float64\n",
        "Briefing_Forecast    float64\n",
        "Market_Expects       float64\n",
        "Prior                float64\n",
        "Revised_From         float64\n",
        "dtype: object"
       ]
      }
     ],
     "prompt_number": 163
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# df_pop3.head()\n",
      "df = pd.concat([df_pop1, df_pop2, df_pop3], axis=1)\n",
      "df.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Date</th>\n",
        "      <th>Time(ET)</th>\n",
        "      <th>Statistic</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td>          Initial Claims</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td>       Continuing Claims</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td> 10:00 AM</td>\n",
        "      <td>   Construction Spending</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td> 10:00 AM</td>\n",
        "      <td>               ISM Index</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td> 10:30 AM</td>\n",
        "      <td> Natural Gas Inventories</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 152,
       "text": [
        "    Date  Time(ET)                Statistic\n",
        "0  Jan 2   8:30 AM           Initial Claims\n",
        "1  Jan 2   8:30 AM        Continuing Claims\n",
        "2  Jan 2  10:00 AM    Construction Spending\n",
        "3  Jan 2  10:00 AM                ISM Index\n",
        "4  Jan 2  10:30 AM  Natural Gas Inventories"
       ]
      }
     ],
     "prompt_number": 152
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# df.ForActual.ix[3]\n",
      "# df_tmp = df.astype(float)\n",
      "df = pd.concat([df, df_tmp], axis=1)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 153
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df[df.Date=='Jan 14']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Date</th>\n",
        "      <th>Time(ET)</th>\n",
        "      <th>Statistic</th>\n",
        "      <th>ForActual</th>\n",
        "      <th>Briefing_Forecast</th>\n",
        "      <th>Market_Expects</th>\n",
        "      <th>Prior</th>\n",
        "      <th>Revised_From</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>28</th>\n",
        "      <td> Jan 14</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td>         Retail Sales</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> 0.3</td>\n",
        "      <td> 0.0</td>\n",
        "      <td> 0.4</td>\n",
        "      <td> 0.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>29</th>\n",
        "      <td> Jan 14</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td> Retail Sales ex-auto</td>\n",
        "      <td> 0.7</td>\n",
        "      <td> 0.1</td>\n",
        "      <td> 0.4</td>\n",
        "      <td> 0.1</td>\n",
        "      <td> 0.4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>30</th>\n",
        "      <td> Jan 14</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td> Export Prices ex-ag.</td>\n",
        "      <td> 0.3</td>\n",
        "      <td> 0.0</td>\n",
        "      <td> 0.0</td>\n",
        "      <td> 0.1</td>\n",
        "      <td> 0.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>31</th>\n",
        "      <td> Jan 14</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td> Import Prices ex-oil</td>\n",
        "      <td> 0.1</td>\n",
        "      <td> 0.0</td>\n",
        "      <td> 0.0</td>\n",
        "      <td> 0.0</td>\n",
        "      <td> 0.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>32</th>\n",
        "      <td> Jan 14</td>\n",
        "      <td> 10:00 AM</td>\n",
        "      <td> Business Inventories</td>\n",
        "      <td> 0.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> 0.3</td>\n",
        "      <td> 0.8</td>\n",
        "      <td> 0.7</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 155,
       "text": [
        "      Date  Time(ET)             Statistic  ForActual  Briefing_Forecast  \\\n",
        "28  Jan 14   8:30 AM          Retail Sales        0.2                0.3   \n",
        "29  Jan 14   8:30 AM  Retail Sales ex-auto        0.7                0.1   \n",
        "30  Jan 14   8:30 AM  Export Prices ex-ag.        0.3                0.0   \n",
        "31  Jan 14   8:30 AM  Import Prices ex-oil        0.1                0.0   \n",
        "32  Jan 14  10:00 AM  Business Inventories        0.4                0.2   \n",
        "\n",
        "    Market_Expects  Prior  Revised_From  \n",
        "28             0.0    0.4           0.7  \n",
        "29             0.4    0.1           0.4  \n",
        "30             0.0    0.1           0.0  \n",
        "31             0.0    0.0           0.0  \n",
        "32             0.3    0.8           0.7  "
       ]
      }
     ],
     "prompt_number": 155
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['label'] = np.where(df['ForActual'] > (df['Briefing_Forecast']+df['Market_Expects'])/2, 1, 0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 164
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Date</th>\n",
        "      <th>Time_ET</th>\n",
        "      <th>Statistic</th>\n",
        "      <th>ForActual</th>\n",
        "      <th>Briefing_Forecast</th>\n",
        "      <th>Market_Expects</th>\n",
        "      <th>Prior</th>\n",
        "      <th>Revised_From</th>\n",
        "      <th>label</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td>          Initial Claims</td>\n",
        "      <td>  339</td>\n",
        "      <td>  340.0</td>\n",
        "      <td>  333.0</td>\n",
        "      <td>  341.0</td>\n",
        "      <td>  338.0</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td>  8:30 AM</td>\n",
        "      <td>       Continuing Claims</td>\n",
        "      <td> 2833</td>\n",
        "      <td> 2875.0</td>\n",
        "      <td> 2875.0</td>\n",
        "      <td> 2931.0</td>\n",
        "      <td> 2923.0</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td> 10:00 AM</td>\n",
        "      <td>   Construction Spending</td>\n",
        "      <td>    1</td>\n",
        "      <td>    0.5</td>\n",
        "      <td>    0.8</td>\n",
        "      <td>    0.9</td>\n",
        "      <td>    0.8</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td> 10:00 AM</td>\n",
        "      <td>               ISM Index</td>\n",
        "      <td>   57</td>\n",
        "      <td>   56.7</td>\n",
        "      <td>   56.9</td>\n",
        "      <td>   57.3</td>\n",
        "      <td>    0.0</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> Jan 2</td>\n",
        "      <td> 10:30 AM</td>\n",
        "      <td> Natural Gas Inventories</td>\n",
        "      <td>    0</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>    0.0</td>\n",
        "      <td>  177.0</td>\n",
        "      <td>    0.0</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 165,
       "text": [
        "    Date   Time_ET                Statistic  ForActual  Briefing_Forecast  \\\n",
        "0  Jan 2   8:30 AM           Initial Claims        339              340.0   \n",
        "1  Jan 2   8:30 AM        Continuing Claims       2833             2875.0   \n",
        "2  Jan 2  10:00 AM    Construction Spending          1                0.5   \n",
        "3  Jan 2  10:00 AM                ISM Index         57               56.7   \n",
        "4  Jan 2  10:30 AM  Natural Gas Inventories          0                0.0   \n",
        "\n",
        "   Market_Expects   Prior  Revised_From  label  \n",
        "0           333.0   341.0         338.0      1  \n",
        "1          2875.0  2931.0        2923.0      0  \n",
        "2             0.8     0.9           0.8      1  \n",
        "3            56.9    57.3           0.0      1  \n",
        "4             0.0   177.0           0.0      0  "
       ]
      }
     ],
     "prompt_number": 165
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def to_number(s):\n",
      "    try:\n",
      "        s1 = float(s)\n",
      "        return s1\n",
      "    except ValueError:\n",
      "        return s\n",
      "    \n",
      "converted = df.ForActual.apply(lambda f : to_number(f)) \n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 78
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "converted.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 79,
       "text": [
        "0     339\n",
        "1    2833\n",
        "2       1\n",
        "3      57\n",
        "4        \n",
        "Name: ForActual, dtype: object"
       ]
      }
     ],
     "prompt_number": 79
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# from sqlalchemy import create_engine\n",
      "# engine = create_engine('postgresql://ethancheung@localhost:5432/test')\n",
      "# engine.connect()\n",
      "# table_name = 'stocknews_yahoocalendar'\n",
      "\n",
      "# df.to_sql(table_name, engine)\n",
      "\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import requests"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 166
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "initial_claims = 'http://www.dol.gov/opa/media/press/opa/'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 167
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "requests.get(initial_claims)"
     ]
    }
   ],
   "metadata": {}
  }
 ]
}