corona-feed.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "from datetime import datetime\nimport pandas as pd\nimport wikipedia as wp\nimport os # for creating the file path\nimport requests\nfrom bs4 import BeautifulSoup\n\ntimezoneLondon = pytz.timezone('Europe/London')\ndatetimeLondon = datetime.now(timezoneLondon)\nexecutionTime = datetimeLondon.strftime(\"_gmt_%d%m%Y_%H%M%S\")\ndirectory = './csv/'\nfileName = directory+\"coronavirus_outbreak_by_country_and_territory\"+executionTime+\".csv\"\n\nif not os.path.isdir(directory):\n    os.mkdir(directory) \n\n# Get the data table source\ndataTable = wp.page(\"2019–20_coronavirus_outbreak_by_country_and_territory\").html().encode(\"UTF-8\")\ndf = pd.read_html(dataTable)[1]\ndf.to_csv(fileName,header=None,index=False)\n\nprint (executionTime)\nprint (fileName)\nprint (df)\n\n\n# This is to get data for UK, can be automated for each country on the table above\n# This part is taken from https://johnricco.github.io/2017/04/04/python-html/ - This deals with the row-span and col-span in html\n# Added replace('\\n','', regex=True) to remove extra carriage return from every cells\nclass html_tables(object):\n    \n    def __init__(self, url):\n        \n        self.url      = url\n        self.r        = requests.get(self.url)\n        self.url_soup = BeautifulSoup(self.r.text)\n        \n    def read(self):\n        \n        self.tables      = []\n        self.tables_html = self.url_soup.find_all(\"table\")\n        \n        # Parse each table\n        for n in range(0, len(self.tables_html)):\n            \n            n_cols = 0\n            n_rows = 0\n            \n            for row in self.tables_html[n].find_all(\"tr\"):\n                col_tags = row.find_all([\"td\", \"th\"])\n                if len(col_tags) > 0:\n                    n_rows += 1\n                    if len(col_tags) > n_cols:\n                        n_cols = len(col_tags)\n            \n            # Create dataframe\n            df = pd.DataFrame(index = range(0, n_rows), columns = range(0, n_cols))\n            \n            # Create list to store rowspan values \n            skip_index = [0 for i in range(0, n_cols)]\n            \n            # Start by iterating over each row in this table...\n            row_counter = 0\n            for row in self.tables_html[n].find_all(\"tr\"):\n                \n                # Skip row if it's blank\n                if len(row.find_all([\"td\", \"th\"])) == 0:\n                    next\n                \n                else:\n                    \n                    # Get all cells containing data in this row\n                    columns = row.find_all([\"td\", \"th\"])\n                    col_dim = []\n                    row_dim = []\n                    col_dim_counter = -1\n                    row_dim_counter = -1\n                    col_counter = -1\n                    this_skip_index = copy.deepcopy(skip_index)\n                    \n                    for col in columns:\n                        \n                        # Determine cell dimensions\n                        colspan = col.get(\"colspan\")\n                        if colspan is None:\n                            col_dim.append(1)\n                        else:\n                            col_dim.append(int(colspan))\n                        col_dim_counter += 1\n                            \n                        rowspan = col.get(\"rowspan\")\n                        if rowspan is None:\n                            row_dim.append(1)\n                        else:\n                            row_dim.append(int(rowspan))\n                        row_dim_counter += 1\n                            \n                        # Adjust column counter\n                        if col_counter == -1:\n                            col_counter = 0  \n                        else:\n                            col_counter = col_counter + col_dim[col_dim_counter - 1]\n                            \n                        while skip_index[col_counter] > 0:\n                            col_counter += 1\n\n                        # Get cell contents  \n                        cell_data = col.get_text()\n                        \n                        # Insert data into cell\n                        df.iat[row_counter, col_counter] = cell_data\n\n                        # Record column skipping index\n                        if row_dim[row_dim_counter] > 1:\n                            this_skip_index[col_counter] = row_dim[row_dim_counter]\n                \n                # Adjust row counter \n                row_counter += 1\n                \n                # Adjust column skipping index\n                skip_index = [i - 1 if i > 0 else i for i in this_skip_index]\n\n            # Append dataframe to list of tables\n            self.tables.append(df.replace('\\n','', regex=True))\n        \n        return(self.tables)\n\ndatetimeLondon = datetime.now(timezoneLondon)\nexecutionTime = datetimeLondon.strftime(\"_gmt_%d%m%Y_%H%M%S\")\nfileName = directory+\"coronavirus_outbreak_UK\"+executionTime+\".csv\"\n\nurl = \"https://en.wikipedia.org/wiki/2020_coronavirus_outbreak_in_the_United_Kingdom\"\nukWiki = html_tables(url)\ndataTable = ukWiki.read()[4]\ndataTable.to_csv(fileName,header=None,index=False)\n\n#print(first_table)",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": "_gmt_06032020_220929\n./csv/coronavirus_outbreak_by_country_and_territory_gmt_06032020_220929.csv\n                                                    0       1          2  \\\n0                             Country or territory[a]   Cases     Deaths   \n1                             China (mainland)[24][c]   80573       3042   \n2                                     South Korea[24]    6593         42   \n3                                            Iran[24]    4747        124   \n4                                           Italy[25]    4636        197   \n5                     International conveyance[24][d]     696          6   \n6                                         Germany[24]     670          0   \n7                                          France[24]     653          9   \n8                                           Japan[24]     420          6   \n9                                           Spain[24]     400          5   \n10                                  United States[24]     260         14   \n11                                    Switzerland[24]     214          1   \n12                                 United Kingdom[26]     163          2   \n13                                      Singapore[24]     130          0   \n14                                    Netherlands[24]     128          1   \n15                                        Belgium[24]     109          0   \n16                                         Norway[24]     108          0   \n17                                      Hong Kong[24]     106          2   \n18                                         Sweden[24]     101          0   \n19                                       Malaysia[24]      83          0   \n20                                      Australia[27]      63          2   \n21                                        Bahrain[24]      60          0   \n22                                         Kuwait[24]      58          0   \n23                                        Austria[24]      55          0   \n24                                       Thailand[24]      48          1   \n25                                         Canada[24]      49          0   \n26                                         Greece[24]      45          0   \n27                                         Taiwan[24]      45          1   \n28                                        Iceland[24]      43          0   \n29                                           Iraq[24]      40          2   \n..                                                ...     ...        ...   \n70                                        Morocco[24]       2          0   \n71                                    Afghanistan[24]       1          0   \n72                                        Andorra[24]       1          0   \n73                                        Armenia[24]       1          0   \n74                                         Bhutan[24]       1          0   \n75                                       Cameroon[24]       1          0   \n76                                       Colombia[49]       1          0   \n77                                     Costa Rica[24]       1          0   \n78                             Dominican Republic[24]       1          0   \n79                                         Jordan[24]       1          0   \n80                                         Latvia[24]       1          0   \n81                                  Liechtenstein[24]       1          0   \n82                                      Lithuania[24]       1          0   \n83                                         Monaco[24]       1          0   \n84                                        Nigeria[24]       1          0   \n85                                North Macedonia[24]       1          0   \n86                                           Peru[50]       1          0   \n87                                   South Africa[24]       1          0   \n88                                         Serbia[51]       1          0   \n89                                       Slovakia[24]       1          0   \n90                                           Togo[52]       1          0   \n91                                        Tunisia[24]       1          0   \n92                                        Ukraine[24]       1          0   \n93                                   Vatican City[24]       1          0   \n94                                       Cambodia[24]       1          0   \n95                                          Nepal[24]       1          0   \n96                                      Sri Lanka[24]       1          0   \n97                                     97 territories  101618  3,460[24]   \n98    As of 6 March 2020 Â· History: China, othersvte     NaN        NaN   \n99  Notes ^ Region where case was diagnosed. Natio...     NaN        NaN   \n\n                3  \n0   Recoveries[b]  \n1           53714  \n2             108  \n3             739  \n4             523  \n5             100  \n6              17  \n7              12  \n8              46  \n9               4  \n10              8  \n11              3  \n12              8  \n13             82  \n14              0  \n15              1  \n16              0  \n17             43  \n18              1  \n19             23  \n20             21  \n21              1  \n22              0  \n23              1  \n24             31  \n25              8  \n26              0  \n27             12  \n28              0  \n29              0  \n..            ...  \n70              0  \n71              0  \n72              0  \n73              0  \n74              0  \n75              0  \n76              0  \n77              0  \n78              0  \n79              0  \n80              0  \n81              0  \n82              0  \n83              0  \n84              0  \n85              0  \n86              0  \n87              0  \n88              0  \n89              0  \n90              0  \n91              0  \n92              0  \n93              0  \n94              1  \n95              1  \n96              1  \n97     55,863[24]  \n98            NaN  \n99            NaN  \n\n[100 rows x 4 columns]\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python36",
      "display_name": "Python 3.6",
      "language": "python"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6",
      "file_extension": ".py",
      "codemirror_mode": {
        "version": 3,
        "name": "ipython"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}