weather_history_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from io import StringIO\n",
    "import pandas as pd\n",
    "\n",
    "url = 'http://meteocenter.net/forecast/all.php'\n",
    "station_id = '36870' #Almaty\n",
    "\n",
    "data_m = [str(x).zfill(2) for x in range(1,13)] # list '01', '02' .. '12'\n",
    "data_y = ['2017', '2018', '2019']\n",
    "forecast_list = []\n",
    "\n",
    "for y in data_y:\n",
    "    for m in data_m:\n",
    "        data = {'Date[0]' : '01', 'Date[1]' : m, 'Date[2]' : y, 'CSV' : '1', 'Station' : station_id}\n",
    "        r = requests.post(url=url, data=data)\n",
    "        if r.status_code != 200:\n",
    "            print('something wrong! with '+ m + y)\n",
    "        \n",
    "        soup = BeautifulSoup(r.text)\n",
    "        txt_forecast = soup.pre.get_text()\n",
    "        \n",
    "        #removing header in every chunnk except first\n",
    "        if (m != data_m[0]) | (y != data_y[0]): \n",
    "            s1 = txt_forecast[txt_forecast.find(\"\\n\")+1:]\n",
    "            txt_forecast = s1[s1.find(\"\\n\")+1:]\n",
    "        \n",
    "        forecast_list.append(txt_forecast.replace('\\r', ''))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Время</th>\n",
       "      <th>Дата</th>\n",
       "      <th>Напр.ветра</th>\n",
       "      <th>Ск.ветра</th>\n",
       "      <th>Видим.</th>\n",
       "      <th>Явл.</th>\n",
       "      <th>Обл.</th>\n",
       "      <th>Форма обл.</th>\n",
       "      <th>Т</th>\n",
       "      <th>Тd</th>\n",
       "      <th>...</th>\n",
       "      <th>Тe</th>\n",
       "      <th>Pmin</th>\n",
       "      <th>dP</th>\n",
       "      <th>Po</th>\n",
       "      <th>Тmin</th>\n",
       "      <th>Tmax</th>\n",
       "      <th>R</th>\n",
       "      <th>s</th>\n",
       "      <th>Сост.почвы</th>\n",
       "      <th>Unnamed: 20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>01.01.17</td>\n",
       "      <td>тихо</td>\n",
       "      <td>{02}</td>\n",
       "      <td>н/о</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10/3 600 м кучево-дожд., верхняя</td>\n",
       "      <td>[Cb cap|Ci sp]</td>\n",
       "      <td>-0.6</td>\n",
       "      <td>-2.9</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1025.4</td>\n",
       "      <td>1.1</td>\n",
       "      <td>923.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>01.01.17</td>\n",
       "      <td>тихо</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10 км</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10/8 600 м кучево-дожд., верхняя</td>\n",
       "      <td>[Cb cap|Ci sp]</td>\n",
       "      <td>0.3</td>\n",
       "      <td>-3.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1027.2</td>\n",
       "      <td>1.9</td>\n",
       "      <td>925.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>равномерныймокрый илистарый снег100%</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>01.01.17</td>\n",
       "      <td>тихо</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10 км</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1/1 1000 м</td>\n",
       "      <td>[Cu hum]</td>\n",
       "      <td>3.3</td>\n",
       "      <td>-3.4</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1026.6</td>\n",
       "      <td>0.5</td>\n",
       "      <td>925.7</td>\n",
       "      <td>-2.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>01.01.17</td>\n",
       "      <td>010</td>\n",
       "      <td>2</td>\n",
       "      <td>4000 м</td>\n",
       "      <td>дым</td>\n",
       "      <td>ясно</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-2.5</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1025.4</td>\n",
       "      <td>-0.8</td>\n",
       "      <td>924.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12</td>\n",
       "      <td>01.01.17</td>\n",
       "      <td>тихо</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10 км</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ясно</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.4</td>\n",
       "      <td>-1.8</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1026.2</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>924.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8995</th>\n",
       "      <td>15</td>\n",
       "      <td>31.12.19</td>\n",
       "      <td>тихо</td>\n",
       "      <td>NaN</td>\n",
       "      <td>н/о</td>\n",
       "      <td>туман</td>\n",
       "      <td>?/?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.9</td>\n",
       "      <td>-1.9</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1026.4</td>\n",
       "      <td>1.1</td>\n",
       "      <td>924.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8996</th>\n",
       "      <td>18</td>\n",
       "      <td>31.12.19</td>\n",
       "      <td>тихо</td>\n",
       "      <td>NaN</td>\n",
       "      <td>н/о</td>\n",
       "      <td>туман</td>\n",
       "      <td>?/?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.9</td>\n",
       "      <td>-1.9</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1026.3</td>\n",
       "      <td>-0.1</td>\n",
       "      <td>924.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8997</th>\n",
       "      <td>21</td>\n",
       "      <td>31.12.19</td>\n",
       "      <td>тихо</td>\n",
       "      <td>NaN</td>\n",
       "      <td>н/о</td>\n",
       "      <td>туман</td>\n",
       "      <td>?/?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.7</td>\n",
       "      <td>-1.9</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1026.1</td>\n",
       "      <td>-0.1</td>\n",
       "      <td>923.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8998</th>\n",
       "      <td>0</td>\n",
       "      <td>01.01.20</td>\n",
       "      <td>тихо</td>\n",
       "      <td>{02}</td>\n",
       "      <td>н/о</td>\n",
       "      <td>туман</td>\n",
       "      <td>?/?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.1</td>\n",
       "      <td>-2.1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1025.7</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>923.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8999</th>\n",
       "      <td>3</td>\n",
       "      <td>01.01.20</td>\n",
       "      <td>тихо</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0200 м</td>\n",
       "      <td>туман</td>\n",
       "      <td>?/?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.1</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>1026.7</td>\n",
       "      <td>0.5</td>\n",
       "      <td>923.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13</td>\n",
       "      <td>равномерныйсухой снег100%</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9000 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Время      Дата Напр.ветра Ск.ветра  Видим.   Явл.  \\\n",
       "0         0  01.01.17       тихо     {02}     н/о    NaN   \n",
       "1         3  01.01.17       тихо      NaN   10 км    NaN   \n",
       "2         6  01.01.17       тихо      NaN   10 км    NaN   \n",
       "3         9  01.01.17        010        2  4000 м    дым   \n",
       "4        12  01.01.17       тихо      NaN   10 км    NaN   \n",
       "...     ...       ...        ...      ...     ...    ...   \n",
       "8995     15  31.12.19       тихо      NaN     н/о  туман   \n",
       "8996     18  31.12.19       тихо      NaN     н/о  туман   \n",
       "8997     21  31.12.19       тихо      NaN     н/о  туман   \n",
       "8998      0  01.01.20       тихо     {02}     н/о  туман   \n",
       "8999      3  01.01.20       тихо      NaN  0200 м  туман   \n",
       "\n",
       "                                  Обл.      Форма обл.    Т   Тd  ...   Тe  \\\n",
       "0     10/3 600 м кучево-дожд., верхняя  [Cb cap|Ci sp] -0.6 -2.9  ... -1.0   \n",
       "1     10/8 600 м кучево-дожд., верхняя  [Cb cap|Ci sp]  0.3 -3.5  ...  0.0   \n",
       "2                           1/1 1000 м        [Cu hum]  3.3 -3.4  ...  3.0   \n",
       "3                                 ясно             NaN  4.0 -2.5  ...  4.0   \n",
       "4                                 ясно             NaN  0.4 -1.8  ...  0.0   \n",
       "...                                ...             ...  ...  ...  ...  ...   \n",
       "8995                              ?/?              NaN -0.9 -1.9  ... -1.0   \n",
       "8996                              ?/?              NaN -0.9 -1.9  ... -1.0   \n",
       "8997                              ?/?              NaN -0.7 -1.9  ... -1.0   \n",
       "8998                              ?/?              NaN -1.1 -2.1  ... -1.0   \n",
       "8999                              ?/?              NaN -2.1 -3.0  ... -2.0   \n",
       "\n",
       "        Pmin   dP     Po  Тmin  Tmax    R    s  \\\n",
       "0     1025.4  1.1  923.3   NaN   NaN  NaN  NaN   \n",
       "1     1027.2  1.9  925.2   NaN   NaN  NaN   11   \n",
       "2     1026.6  0.5  925.7  -2.9   NaN  NaN  NaN   \n",
       "3     1025.4 -0.8  924.9   NaN   NaN  NaN  NaN   \n",
       "4     1026.2 -0.5  924.4   NaN   NaN  NaN  NaN   \n",
       "...      ...  ...    ...   ...   ...  ...  ...   \n",
       "8995  1026.4  1.1  924.1   NaN   0.3  NaN  NaN   \n",
       "8996  1026.3 -0.1  924.0   NaN   0.3  0.0  NaN   \n",
       "8997  1026.1 -0.1  923.9   NaN   NaN  NaN  NaN   \n",
       "8998  1025.7 -0.5  923.4   NaN   NaN  NaN  NaN   \n",
       "8999  1026.7  0.5  923.9   NaN   NaN  NaN   13   \n",
       "\n",
       "                                Сост.почвы Unnamed: 20  \n",
       "0                                      NaN         NaN  \n",
       "1     равномерныймокрый илистарый снег100%         NaN  \n",
       "2                                      NaN         NaN  \n",
       "3                                      NaN         NaN  \n",
       "4                                      NaN         NaN  \n",
       "...                                    ...         ...  \n",
       "8995                                   NaN         NaN  \n",
       "8996                                   NaN         NaN  \n",
       "8997                                   NaN         NaN  \n",
       "8998                                   NaN         NaN  \n",
       "8999             равномерныйсухой снег100%         NaN  \n",
       "\n",
       "[9000 rows x 21 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = pd.read_csv(StringIO(''.join(forecast_list)), sep =';', encoding='cp1251', header = 0, skip_blank_lines =True)\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}