ЯндексПогода парсер pandas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cabb35ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "from statistics import mean \n",
    "import pandas as pd\n",
    "import re\n",
    "from datetime import datetime,timedelta \n",
    "import requests\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "39f1af75",
   "metadata": {},
   "outputs": [],
   "source": [
    "#даты\n",
    "date_now=datetime.today().date()\n",
    "day_now=date_now.day #<--- день сегодня\n",
    "date_tomorrow=date_now + timedelta(days=1)\n",
    "day_tomorrow=(date_now + timedelta(days=1)).day #<--- день завтра\n",
    "day_after_tomorrow=(date_now + timedelta(days=2)).day #<--- день послезавтра"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "af63bf67",
   "metadata": {},
   "outputs": [],
   "source": [
    "#считываем страницу с прогнозом Яндекса через read_html, получаем необработанные данные \n",
    "ya_df=pd.read_html('https://yandex.ru/pogoda/details?lat=55.753215&lon=37.622504&via=ms')\n",
    "\n",
    "#каждый день находится в отдельной ячейке, прилепим их их друг под другом\n",
    "ya_new=pd.concat([ya_df[0],ya_df[1],ya_df[2],ya_df[3],ya_df[4],ya_df[5],ya_df[6],ya_df[7],ya_df[8],ya_df[9]],ignore_index=True)\n",
    "ya_new.columns=['temp','nan','weather','press','wet','wind','t_eff']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a3ccf8b",
   "metadata": {},
   "source": [
    "## ЯНДЕКС"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "33e61351",
   "metadata": {},
   "outputs": [],
   "source": [
    "#поработаем со столбцом температуры при помощи регулярных выражений\n",
    "temp_clean=[]\n",
    "for i in range(0,ya_new.shape[0]):\n",
    "    t_str=ya_new['temp'][i]\n",
    "    temp_str=re.search(r'[+ -]\\d{1,2}\\…[+ -]\\d{1,2}',t_str)\n",
    "    temp_clean.append(temp_str.group(0))\n",
    "    \n",
    "temp_fin=pd.Series(temp_clean) \n",
    "temp_fin=temp_fin.str.rsplit('…',expand=True)\n",
    "\n",
    "temp_fin.columns=['t_down','t_up']   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2d139640",
   "metadata": {},
   "outputs": [],
   "source": [
    "#поработаем со столбцом ветра\n",
    "\n",
    "wind_clean=[]\n",
    "for i in range(0,ya_new.shape[0]):\n",
    "    w_str=ya_new['wind'][i]\n",
    "    wind_str=re.search(r'\\d{1,2},\\d',w_str)\n",
    "    wind_clean.append(wind_str.group(0))\n",
    "    \n",
    "wind_fin=pd.Series(wind_clean)  \n",
    "wind_fin=wind_fin.str.replace(',','.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a9ab25c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#поработаем со столбцом эффективной температуры\n",
    "t_eff=[]\n",
    "for i in range(0,ya_new.shape[0]):\n",
    "    string=ya_new['t_eff'][i]\n",
    "    if string[0]=='−':\n",
    "        s_1=re.sub('[^0-9]','', string)\n",
    "        #end=(len(s_1)/2)\n",
    "        s_fin='−'+s_1[:len(s_1)//2]\n",
    "    if string[0] != '−':  \n",
    "        s_1=re.sub('[^0-9]','', string)\n",
    "        s_fin=s_1[:len(s_1)//2]\n",
    "    t_eff.append(s_fin)\n",
    "    \n",
    "temp_eff=pd.Series(t_eff) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dddbf455",
   "metadata": {},
   "outputs": [],
   "source": [
    "#собираем в табличку\n",
    "list_days=[10*['утро','день','вечер','ночь']]\n",
    "list_days=pd.DataFrame(list_days).T\n",
    "\n",
    "ya_full=pd.concat([list_days,temp_fin,ya_new[['press','wet']],wind_fin,temp_eff,ya_new['weather']],axis=1)\n",
    "\n",
    "ya_full.columns=['time','t_down','t_up','press','wet','wind','t_eff','weather']\n",
    "\n",
    "#убираем \"грязь\"\n",
    "ya_full['wet']=ya_full['wet'].str.replace('%','')\n",
    "ya_full['t_down']=ya_full['t_down'].str.replace('−','-')\n",
    "ya_full['t_up']=ya_full['t_up'].str.replace('−','-')\n",
    "ya_full['t_eff']=ya_full['t_eff'].str.replace('−','-')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fef2b93e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>t_down</th>\n",
       "      <th>t_up</th>\n",
       "      <th>press</th>\n",
       "      <th>wet</th>\n",
       "      <th>wind</th>\n",
       "      <th>t_eff</th>\n",
       "      <th>weather</th>\n",
       "      <th>t_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>утро</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>755</td>\n",
       "      <td>55</td>\n",
       "      <td>2.3</td>\n",
       "      <td>7</td>\n",
       "      <td>Ясно</td>\n",
       "      <td>9.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>день</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>754</td>\n",
       "      <td>31</td>\n",
       "      <td>2.3</td>\n",
       "      <td>12</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>14.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>вечер</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>752</td>\n",
       "      <td>43</td>\n",
       "      <td>2.0</td>\n",
       "      <td>11</td>\n",
       "      <td>Пасмурно</td>\n",
       "      <td>13.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ночь</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>752</td>\n",
       "      <td>66</td>\n",
       "      <td>2.1</td>\n",
       "      <td>5</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>8.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>утро</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>752</td>\n",
       "      <td>56</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>9.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>день</td>\n",
       "      <td>11</td>\n",
       "      <td>13</td>\n",
       "      <td>751</td>\n",
       "      <td>45</td>\n",
       "      <td>3.5</td>\n",
       "      <td>8</td>\n",
       "      <td>Небольшой дождь</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>вечер</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>751</td>\n",
       "      <td>68</td>\n",
       "      <td>4.2</td>\n",
       "      <td>2</td>\n",
       "      <td>Небольшой дождь</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ночь</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>750</td>\n",
       "      <td>83</td>\n",
       "      <td>4.6</td>\n",
       "      <td>-2</td>\n",
       "      <td>Пасмурно</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>утро</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>752</td>\n",
       "      <td>78</td>\n",
       "      <td>6.5</td>\n",
       "      <td>-4</td>\n",
       "      <td>Пасмурно</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>день</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>752</td>\n",
       "      <td>68</td>\n",
       "      <td>6.9</td>\n",
       "      <td>-3</td>\n",
       "      <td>Пасмурно</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>вечер</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>753</td>\n",
       "      <td>77</td>\n",
       "      <td>5.5</td>\n",
       "      <td>-2</td>\n",
       "      <td>Пасмурно</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ночь</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>754</td>\n",
       "      <td>80</td>\n",
       "      <td>3.7</td>\n",
       "      <td>-3</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>утро</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>755</td>\n",
       "      <td>75</td>\n",
       "      <td>4.2</td>\n",
       "      <td>-2</td>\n",
       "      <td>Малооблачно</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>день</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>756</td>\n",
       "      <td>52</td>\n",
       "      <td>4.9</td>\n",
       "      <td>3</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>7.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>вечер</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>757</td>\n",
       "      <td>60</td>\n",
       "      <td>3.1</td>\n",
       "      <td>4</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ночь</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>757</td>\n",
       "      <td>80</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>утро</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>758</td>\n",
       "      <td>71</td>\n",
       "      <td>2.9</td>\n",
       "      <td>2</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>день</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>758</td>\n",
       "      <td>47</td>\n",
       "      <td>3.3</td>\n",
       "      <td>8</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>вечер</td>\n",
       "      <td>9</td>\n",
       "      <td>12</td>\n",
       "      <td>757</td>\n",
       "      <td>50</td>\n",
       "      <td>2.4</td>\n",
       "      <td>7</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>10.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ночь</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>757</td>\n",
       "      <td>67</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2</td>\n",
       "      <td>Облачно с прояснениями</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     time  t_down  t_up  press  wet  wind  t_eff                 weather  \\\n",
       "0    утро       6    13    755   55   2.3      7                    Ясно   \n",
       "1    день      14    15    754   31   2.3     12  Облачно с прояснениями   \n",
       "2   вечер      12    15    752   43   2.0     11                Пасмурно   \n",
       "3    ночь       7    10    752   66   2.1      5  Облачно с прояснениями   \n",
       "4    утро       7    12    752   56   3.0      7  Облачно с прояснениями   \n",
       "5    день      11    13    751   45   3.5      8         Небольшой дождь   \n",
       "6   вечер       5     9    751   68   4.2      2         Небольшой дождь   \n",
       "7    ночь       1     5    750   83   4.6     -2                Пасмурно   \n",
       "8    утро       1     4    752   78   6.5     -4                Пасмурно   \n",
       "9    день       4     5    752   68   6.9     -3                Пасмурно   \n",
       "10  вечер       3     4    753   77   5.5     -2                Пасмурно   \n",
       "11   ночь       1     3    754   80   3.7     -3  Облачно с прояснениями   \n",
       "12   утро       1     4    755   75   4.2     -2             Малооблачно   \n",
       "13   день       7     8    756   52   4.9      3  Облачно с прояснениями   \n",
       "14  вечер       7     9    757   60   3.1      4  Облачно с прояснениями   \n",
       "15   ночь       3     4    757   80   2.0      1  Облачно с прояснениями   \n",
       "16   утро       4     8    758   71   2.9      2  Облачно с прояснениями   \n",
       "17   день      10    12    758   47   3.3      8  Облачно с прояснениями   \n",
       "18  вечер       9    12    757   50   2.4      7  Облачно с прояснениями   \n",
       "19   ночь       4     6    757   67   1.5      2  Облачно с прояснениями   \n",
       "\n",
       "    t_mean  \n",
       "0      9.5  \n",
       "1     14.5  \n",
       "2     13.5  \n",
       "3      8.5  \n",
       "4      9.5  \n",
       "5     12.0  \n",
       "6      7.0  \n",
       "7      3.0  \n",
       "8      2.5  \n",
       "9      4.5  \n",
       "10     3.5  \n",
       "11     2.0  \n",
       "12     2.5  \n",
       "13     7.5  \n",
       "14     8.0  \n",
       "15     3.5  \n",
       "16     6.0  \n",
       "17    11.0  \n",
       "18    10.5  \n",
       "19     5.0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#поработаем с форматами\n",
    "ya_full[['t_down','t_up','wet']]=ya_full[['t_down','t_up','wet']].astype('int')\n",
    "ya_full['wind']=ya_full['wind'].astype('float')\n",
    "ya_full['t_eff']=ya_full['t_eff'].astype('int')\n",
    "\n",
    "#считаем среднюю температуру\n",
    "ya_full['t_mean']=(ya_full['t_up']+ya_full['t_down'])/2\n",
    "\n",
    "ya_full.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb2c506f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vp": {
   "vp_config_version": "1.0.0",
   "vp_menu_width": 273,
   "vp_note_display": false,
   "vp_note_width": 0,
   "vp_position": {
    "width": 278
   },
   "vp_section_display": false,
   "vp_signature": "VisualPython"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}