parser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def html_stripper(text):\n",
    "    return re.sub('<[^<]+?>', '', str(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "page = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ЦАО"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "district = 'http://www.cian.ru/cat.php?deal_type=sale&district%5B0%5D=13&district%5B1%5D=14&district%5B2%5D=15&district%5B3%5D=16&district%5B4%5D=17&district%5B5%5D=18&district%5B6%5D=19&district%5B7%5D=20&district%5B8%5D=21&district%5B9%5D=22&engine_version=2&offer_type=flat&p={}&room1=1&room2=1&room3=1&room4=1&room5=1&room6=1'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Собираем ссылки на все квартиры первых тридцати страниц выдачи"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "links = []\n",
    "\n",
    "for page in range(1, 30):\n",
    "    page_url =  district.format(page)\n",
    "\n",
    "    search_page = requests.get(page_url)\n",
    "    search_page = search_page.content\n",
    "    search_page = BeautifulSoup(search_page, 'lxml')\n",
    "\n",
    "    flat_urls = search_page.findAll('div', attrs = {'ng-class':\"{'serp-item_removed': offer.remove.state, 'serp-item_popup-opened': isPopupOpen}\"})\n",
    "    flat_urls = re.split('http://www.cian.ru/sale/flat/|/\" ng-class=\"', str(flat_urls))\n",
    "\n",
    "    for link in flat_urls:\n",
    "        if link.isdigit():\n",
    "            links.append(link)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cтандартный блок, в котором мы получаем по ссылке текст страницы в удобном формате"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flat_url = 'http://www.cian.ru/sale/flat/' + str(links[0]) + '/'\n",
    "#flat_url = 'http://www.cian.ru/sale/flat/150531912/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Функция для извлечения цены квартиры из странички, взято из лекции"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 567,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getPrice(flat_page):\n",
    "    price = flat_page.find('div', attrs={'class':'object_descr_price'})\n",
    "    price = re.split('<div>|руб|\\W', str(price))\n",
    "    price = \"\".join([i for i in price if i.isdigit()][-4:])\n",
    "    return int(price)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Функция для сбора всех цен с сайта в выбранном диапазоне ссылок ( l < r )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 583,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllPrices(l, r):\n",
    "    prices = []\n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        prices.append(getPrice(flat_page))\n",
    "    return prices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Теперь используем эти функции и составим колонку для всех цен"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 589,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "prices = getAllPrices(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Извлечем координаты так же, как это было продемонстировано на лекции"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 591,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "flat_url = 'http://www.cian.ru/sale/flat/' + str(links[0]) + '/'\n",
    "#flat_url = 'http://www.cian.ru/sale/flat/150531912/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "\n",
    "coords = flat_page.find('div', attrs={'class':'map_info_button_extend'}).contents[1]\n",
    "coords = re.split('&amp|center=|%2C', str(coords))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 592,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<a href=\"http://map.cian.ru/?deal_type=2',\n",
       " ';flats=yes',\n",
       " ';',\n",
       " '55.762586',\n",
       " '37.597809',\n",
       " ';room0=1',\n",
       " ';room1=1',\n",
       " ';room2=1',\n",
       " ';room3=1',\n",
       " ';room4=1\" rel=\"nofollow\" target=\"_blank\"><i></i>Посмотреть предложения рядом</a>']"
      ]
     },
     "execution_count": 592,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 593,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coords_list = []\n",
    "for item in coords:\n",
    "    if item[0].isdigit():\n",
    "        coords_list.append(item)\n",
    "lat = float(coords_list[0])\n",
    "lon = float(coords_list[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 594,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55.762586"
      ]
     },
     "execution_count": 594,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 595,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37.597809"
      ]
     },
     "execution_count": 595,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Определим функцию получения координат из текста страницы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 596,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getCoords_at(flat_page):\n",
    "    coords = flat_page.find('div', attrs={'class':'map_info_button_extend'}).contents[1]\n",
    "    coords = re.split('&amp|center=|%2C', str(coords))\n",
    "    coords_list = []\n",
    "    for item in coords:\n",
    "        if item[0].isdigit():\n",
    "            coords_list.append(item)\n",
    "    lat = float(coords_list[0])\n",
    "    lon = float(coords_list[1])\n",
    "    return lat, lon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### И функцию для получения всех координат в выбранном диапазоне ссылок (такая схема из двух функций будет повторяться везде)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 812,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllCoordinates(l, r):\n",
    "    coordinates = []\n",
    "\n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        coordinates.append(getCoords(flat_page))\n",
    "        \n",
    "    return coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 599,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "coordinates = getAllCoordinates(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Теперь нам предстоит использовать полученные координаты для вычисления расстояния до центра города. Воспользуемся для этого формулой для вычисления расстояния между двумя точками, расположенными на сфере. Запишем отдельную функцию"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 649,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from math import radians, cos, sin, asin, sqrt\n",
    "AVG_EARTH_RADIUS = 6371\n",
    "\n",
    "def haversine(point1, point2):\n",
    "\n",
    "    # извлекаем долготу и широту\n",
    "    lat1, lng1 = point1\n",
    "    lat2, lng2 = point2\n",
    "\n",
    "    # переводим все эти значения в радианы\n",
    "    lat1, lng1, lat2, lng2 = map(radians, (lat1, lng1, lat2, lng2))\n",
    "\n",
    "    # вычисляем расстояние по формуле\n",
    "    lat = lat2 - lat1\n",
    "    lng = lng2 - lng1\n",
    "    d = sin(lat * 0.5) ** 2 + cos(lat1) * cos(lat2) * sin(lng * 0.5) ** 2\n",
    "    h = 2 * AVG_EARTH_RADIUS * asin(sqrt(d))\n",
    "\n",
    "    return h "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 651,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "MSC_POINT_ZERO = (55.755831, 37.617673)\n",
    "distance = []\n",
    "for i in range(0, len(coordinates)):\n",
    "    distance.append(haversine(MSC_POINT_ZERO, coordinates[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rooms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Перейдем к числу комнат, тоже сделано по аналогии с лекционным ноутбуком"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 612,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "flat_url = 'http://www.cian.ru/sale/flat/' + str(links[2]) + '/'\n",
    "#flat_url = 'http://www.cian.ru/sale/flat/150844464/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 613,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n                \\n    \\n        2-комн. кв.\\xa0\\n    \\n\\n                \\n                    \\n                \\n                '"
      ]
     },
     "execution_count": 613,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rooms_n = flat_page.find('div', attrs={'class':'object_descr_title'})\n",
    "rooms_n = html_stripper(rooms_n)\n",
    "rooms_n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 614,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " '                ',\n",
       " '    ',\n",
       " '        2',\n",
       " 'комн. кв.\\xa0',\n",
       " '    ',\n",
       " '',\n",
       " '                ',\n",
       " '                    ',\n",
       " '                ',\n",
       " '                ']"
      ]
     },
     "execution_count": 614,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split('-|\\n', rooms_n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Страницы о многокомнатных квартирах (  > 5 ) не содержат информации о количестве комнат в явном виде, такие квартиры фигурируют как \"многокомнатные\". Так что соответствующие объекты у меня получат в столбец Rooms значение 'mult'. Это более достоверная информация, чем, например, 6."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 617,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getRoom(flat_page):\n",
    "    rooms_n = flat_page.find('div', attrs={'class':'object_descr_title'})\n",
    "    rooms_n = html_stripper(rooms_n)\n",
    "    room_number = ''\n",
    "    flag = 0\n",
    "    for i in re.split('-|\\n', rooms_n):\n",
    "        if 'много' in i:\n",
    "            flag = 1\n",
    "            break\n",
    "        elif 'комн' in i:\n",
    "            break\n",
    "        else:\n",
    "            room_number += i\n",
    "    \n",
    "    if (flag):\n",
    "        room_number = 'mult'\n",
    "    room_number = \"\".join(room_number.split())\n",
    "    return room_number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 623,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getAllRooms(l, r):\n",
    "    rooms = []\n",
    "\n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        rooms.append(getRoom(flat_page))\n",
    "        \n",
    "    return rooms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 625,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rooms = getAllRooms(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrdist и Walking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Теперь начнем разбираться с расстоянием до метро,  заодно и со способом до него добраться (пешком или на машине)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 626,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#flat_url = 'http://www.cian.ru/sale/flat/' + str(links[2]) + '/'\n",
    "flat_url = 'http://www.cian.ru/sale/flat/150387502/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 629,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<div class=\"object_descr_metro\">\\n<p class=\"objects_item_metro_prg\">\\n<span class=\"metro_icon metro_icon_region_1\" style=\"background-color: #F58232\"></span>\\n<a class=\"object_item_',\n",
       " '\" href=\"/cat.php?engine_version=2&amp;metro%5B0%5D=47\" rel=\"nofollow\" target=\"_blank\">Китай-город,</a>\\n<span class=\"object_item_metro_comment\">\\n                10\\n                ',\n",
       " '.\\n                \\n                    пешком\\n                \\n            </span>\\n</p>\\n</div>']"
      ]
     },
     "execution_count": 629,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metro = flat_page.find('div', attrs={'class':'object_descr_metro'})\n",
    "metro = re.split('metro_name|мин', str(metro))\n",
    "metro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 630,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[\\'<div class=\"object_descr_metro\">\\\\n<p class=\"objects_item_metro_prg\">\\\\n<span class=\"metro_icon metro_icon_region_1\" style=\"background-color: #F58232\"></span>\\\\n<a class=\"object_item_\\', \\'\" href=\"/cat.php?engine_version=2&amp;metro%5B0%5D=47\" rel=\"nofollow\" target=\"_blank\">Китай-город,</a>\\\\n<span class=\"object_item_metro_comment\">\\\\n                10\\\\n                \\', \\'.\\\\n                \\\\n                    пешком\\\\n                \\\\n            </span>\\\\n</p>\\\\n</div>\\']']"
      ]
     },
     "execution_count": 630,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split('metro_name|мин', str(metro))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Тут у нас появляется отличие от того, что было раньше: некоторые поля на Циане являются обязательными для заполнения, а некоторые нет, и вот эти поля обязательными не являются. Класть в данные по квартире, по которой нет этой информации, дефолтный 0 не будет правильным, потому что люди могли не указать как 3 минуты пешком (ну мало ли, лень было вписывать что-то в ячейки, квартира и так продастся), а могли не указать 30 минут на машине. Если уравнивать такие вещи (и тем более все их забивать нулями), то позже это плохо скажется на модели. Так что будем класть в такие ячейки nan'ы."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrdist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Функция для извлечения расстояния до метро в минутах"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 640,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getMetroDistance(flat_page):\n",
    "    metro = flat_page.find('div', attrs={'class':'object_descr_metro'})\n",
    "    metro = re.split('metro_name|мин', str(metro))\n",
    "    if (len(metro) > 2): # если оба поля не были заполнены, то предыдущий сплит даст размерность 2\n",
    "\n",
    "        metro_dist = 0\n",
    "        power = 0\n",
    "        \n",
    "        # кусок metro[1] после сплита будет содержать в конце кучу хлама, потом количество минут (если есть)\n",
    "        flag = 0\n",
    "        for i in range(0, len(metro[1])):\n",
    "            if metro[1][-i-1].isdigit():\n",
    "                flag = 1\n",
    "                metro_dist += int(metro[1][-i-1]) * 10 ** power\n",
    "                power += 1\n",
    "            elif (flag == 1):\n",
    "                break\n",
    "    else:\n",
    "        metro_dist = np.nan\n",
    "\n",
    "    return metro_dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 641,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllMetroDistances(l, r):\n",
    "    metro_distance = []\n",
    "\n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        metro_distance.append(getMetroDistance(flat_page))\n",
    "        \n",
    "    return metro_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "metro_distances = getAllMetroDistances(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Walking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Аналогично поступим с колонкой walking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 644,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getMetroWalking(flat_page):\n",
    "    metro = flat_page.find('div', attrs={'class':'object_descr_metro'})\n",
    "    metro = re.split('metro_name|мин', str(metro))\n",
    "    if (len(metro) > 2): # если оба поля не были заполнены, то предыдущий сплит даст размерность 2\n",
    "\n",
    "        if 'пешк' in metro[2]:\n",
    "            walking = 1\n",
    "        elif 'машин' in metro[2]:\n",
    "            walking = 0\n",
    "        else:\n",
    "            walking = np.nan # да, проверка на то, отсутствовали ли оба поля была. мне лично не попадались ситуации, где бы не \n",
    "                             # было заполнено только значение поля \"пешком/на машине\", но вдруг они есть? на такой случай проверка\n",
    "    else:\n",
    "        walking = np.nan\n",
    "\n",
    "    return walking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 645,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllMetroWalking(l, r):\n",
    "    metro_walking = []\n",
    "\n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        metro_walking.append(getMetroWalking(flat_page))\n",
    "        \n",
    "    return metro_walking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 647,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "walking = getAllMetroWalking(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Теперь перейдем к работе с данными, которые мы будем извлекать из таблицы. Это будущие колонки:\n",
    "### Bricks - кирпичный ли дом, монолитный (варианта жб на Циане нет)/иное\n",
    "### New - новостройка/вторичка\n",
    "### Totsp, Livsp, Kitsp - общая, жилая и площадь кухни соответственно\n",
    "### Tel - наличие телефона\n",
    "### Bal - наличие балкона\n",
    "### Floor - этаж \n",
    "### Nfloors - всего этажей в доме."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Среди вышеперечисленного обязательны к заполнению только поля Totsp, Livsp и Floor. В остальных местах могут быть nan'ы."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bricks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 656,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\nОбщая информация:\\n\\n\\nЭтаж:\\n\\n                3\\xa0/\\xa010\\n                \\n            \\n\\n\\nТип дома:\\n\\n            вторичка, \\n            кирпичный\\n        \\n\\n\\nТип продажи:\\nсвободная\\n\\n\\nОбщая площадь:\\n\\n34\\xa0м2\\n\\n\\n\\nПлощадь комнат:\\n\\n19\\xa0м2\\n\\n\\n\\nЖилая площадь:\\n\\n19\\xa0м2\\n\\n\\n\\nПлощадь кухни:\\n\\n6\\xa0м2\\n\\n\\n\\nСовмещенных санузлов:\\n1\\n            \\n\\n\\nБалкон:\\n1 балк.\\n        \\n\\n\\nЛифт:\\n1 пасс.\\n\\n\\nПарковка:\\nназемная\\n\\n\\nТелефон:\\nда\\n\\n\\nВид из окна:\\nдвор\\n\\n\\nРемонт:\\nкосметический\\n\\n'"
      ]
     },
     "execution_count": 656,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#flat_url = 'http://www.cian.ru/sale/flat/' + str(links[2]) + '/'\n",
    "flat_url = 'http://www.cian.ru/sale/flat/150387502/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "table = html_stripper(table)\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Информация по Brick и New содержится между следующими двумя обязательными полями: \"Этаж\" и \"Тип продажи\" (то есть на наличие этих слов в этом месте можно положиться)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 658,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "':\\n\\n                3\\xa0/\\xa010\\n                \\n            \\n\\n\\nТип дома:\\n\\n            вторичка, \\n            кирпичный\\n        \\n\\n\\n'"
      ]
     },
     "execution_count": 658,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "building_block = re.split('Этаж|Тип продажи', table)[1]\n",
    "building_block"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ситуация следующая: поле \"Тип дома\" может быть, его может не быть (тогда brick = nan). Если оно есть, то оно может содержать информацию по полю Brick, а может и не содержать. Поле Brick может задавать два значения: 1- если это кирпичный или монолитный (или кирпично-монолитный), и 0 - если это что-то другое (и мы заранее можем посмотреть на сайте на то, какие еще бывают варианты: панельный, блочный, деревянный и сталинский). Запишем эти размышления в функцию"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 694,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getBrick(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    brick = np.nan\n",
    "    \n",
    "    building_block = re.split('Этаж|Тип продажи', table)[1]\n",
    "    if 'Тип дом' in building_block:\n",
    "        if (('кирпич' in building_block) | ('монолит' in building_block)):\n",
    "            brick = 1\n",
    "        elif (('панельн' in building_block) | ('деревян' in building_block) | ('сталин' in building_block) | \n",
    "              ('блочн' in building_block)):\n",
    "            brick = 0\n",
    "\n",
    "            \n",
    "    return brick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 695,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getAllBricks(l, r):\n",
    "    \n",
    "    bricks = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        bricks.append(getBrick(flat_page))\n",
    "        \n",
    "    return bricks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 697,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bricks = getAllBricks(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Теперь перейдем к New - тут либо один вариант из двух (1 - новостройка, 0 - вторичка), либо nan."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 690,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getNew(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    new = np.nan\n",
    "    \n",
    "    building_block = re.split('Этаж|Тип продажи', table)[1]\n",
    "    if 'Тип дом' in building_block:\n",
    "        if 'новостр' in building_block:\n",
    "            new = 1\n",
    "        elif 'втор' in building_block:\n",
    "            new = 0\n",
    "\n",
    "            \n",
    "    return new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 691,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getAllNew(l, r):\n",
    "    \n",
    "    new = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        new.append(getNew(flat_page))\n",
    "        \n",
    "    return new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 693,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "new = getAllNew(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Теперь переходим дальше"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Floor, Nfloors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 698,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\nОбщая информация:\\n\\n\\nЭтаж:\\n\\n                3\\xa0/\\xa010\\n                \\n            \\n\\n\\nТип дома:\\n\\n            вторичка, \\n            кирпичный\\n        \\n\\n\\nТип продажи:\\nсвободная\\n\\n\\nОбщая площадь:\\n\\n34\\xa0м2\\n\\n\\n\\nПлощадь комнат:\\n\\n19\\xa0м2\\n\\n\\n\\nЖилая площадь:\\n\\n19\\xa0м2\\n\\n\\n\\nПлощадь кухни:\\n\\n6\\xa0м2\\n\\n\\n\\nСовмещенных санузлов:\\n1\\n            \\n\\n\\nБалкон:\\n1 балк.\\n        \\n\\n\\nЛифт:\\n1 пасс.\\n\\n\\nПарковка:\\nназемная\\n\\n\\nТелефон:\\nда\\n\\n\\nВид из окна:\\nдвор\\n\\n\\nРемонт:\\nкосметический\\n\\n'"
      ]
     },
     "execution_count": 698,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#flat_url = 'http://www.cian.ru/sale/flat/' + str(links[2]) + '/'\n",
    "flat_url = 'http://www.cian.ru/sale/flat/150387502/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "table = html_stripper(table)\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Информация по Floor и Nfloors тоже содержится между обязательными полями \"Этаж\" и \"Тип продажи\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 699,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "':\\n\\n                3\\xa0/\\xa010\\n                \\n            \\n\\n\\nТип дома:\\n\\n            вторичка, \\n            кирпичный\\n        \\n\\n\\n'"
      ]
     },
     "execution_count": 699,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "building_block = re.split('Этаж|Тип продажи', table)[1]\n",
    "building_block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 700,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[':',\n",
       " '',\n",
       " '                3',\n",
       " '10',\n",
       " '                ',\n",
       " '            ',\n",
       " '',\n",
       " '',\n",
       " 'Тип дома:',\n",
       " '',\n",
       " '            вторичка, ',\n",
       " '            кирпичный',\n",
       " '        ',\n",
       " '',\n",
       " '',\n",
       " '']"
      ]
     },
     "execution_count": 700,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "floor_block = re.split('\\xa0/\\xa0|\\n|\\xa0', building_block)\n",
    "floor_block"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Floor указан обязательно, и он лежит во второй ячейке, Nfloors может не быть, но если есть - то он в третьей."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Floor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 701,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getFloor(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    floor_is = 0\n",
    "    \n",
    "    building_block = re.split('Этаж|Тип продажи', table)[1]\n",
    "    floor_block = re.split('\\xa0/\\xa0|\\n|\\xa0', building_block)\n",
    "    \n",
    "    for i in range(1, len(floor_block[2]) + 1):\n",
    "        if(floor_block[2][-i].isdigit()):\n",
    "            floor_is += int(floor_block[2][-i]) * 10**(i - 1)\n",
    "            \n",
    "    return floor_is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 702,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllFloors(l, r):\n",
    "    \n",
    "    floors = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        floors.append(getFloor(flat_page))\n",
    "        \n",
    "    return floors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 704,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "floors = getAllFloors(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nfloors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 705,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getNFloor(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    floors_count = np.nan\n",
    "    \n",
    "    building_block = re.split('Этаж|Тип продажи', table)[1]\n",
    "    floor_block = re.split('\\xa0/\\xa0|\\n|\\xa0', building_block)\n",
    "    \n",
    "    if floor_block[3].isdigit():\n",
    "        floors_count = int(floor_block[3])\n",
    "            \n",
    "    return floors_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 706,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllNFloors(l, r):\n",
    "    \n",
    "    nfloors = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        nfloors.append(getNFloor(flat_page))\n",
    "        \n",
    "    return nfloors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 708,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nfloors = getAllNFloors(0, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Перейдем дальше"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Totsp, Livesp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 709,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\nОбщая информация:\\n\\n\\nЭтаж:\\n\\n                3\\xa0/\\xa010\\n                \\n            \\n\\n\\nТип дома:\\n\\n            вторичка, \\n            кирпичный\\n        \\n\\n\\nТип продажи:\\nсвободная\\n\\n\\nОбщая площадь:\\n\\n34\\xa0м2\\n\\n\\n\\nПлощадь комнат:\\n\\n19\\xa0м2\\n\\n\\n\\nЖилая площадь:\\n\\n19\\xa0м2\\n\\n\\n\\nПлощадь кухни:\\n\\n6\\xa0м2\\n\\n\\n\\nСовмещенных санузлов:\\n1\\n            \\n\\n\\nБалкон:\\n1 балк.\\n        \\n\\n\\nЛифт:\\n1 пасс.\\n\\n\\nПарковка:\\nназемная\\n\\n\\nТелефон:\\nда\\n\\n\\nВид из окна:\\nдвор\\n\\n\\nРемонт:\\nкосметический\\n\\n'"
      ]
     },
     "execution_count": 709,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#flat_url = 'http://www.cian.ru/sale/flat/' + str(links[2]) + '/'\n",
    "flat_url = 'http://www.cian.ru/sale/flat/150387502/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "table = html_stripper(table)\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Всех полей, что находятся справа от \"Общая площадь\", может не быть. Так что ограничим нашу область поиска только слева."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 710,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "':\\n\\n34\\xa0м2\\n\\n\\n\\nПлощадь комнат:\\n\\n19\\xa0м2\\n\\n\\n\\nЖилая площадь:\\n\\n19\\xa0м2\\n\\n\\n\\nПлощадь кухни:\\n\\n6\\xa0м2\\n\\n\\n\\nСовмещенных санузлов:\\n1\\n            \\n\\n\\nБалкон:\\n1 балк.\\n        \\n\\n\\nЛифт:\\n1 пасс.\\n\\n\\nПарковка:\\nназемная\\n\\n\\nТелефон:\\nда\\n\\n\\nВид из окна:\\nдвор\\n\\n\\nРемонт:\\nкосметический\\n\\n'"
      ]
     },
     "execution_count": 710,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "space_block = re.split('Общая площадь', table)[1]\n",
    "space_block"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Totsp и Livesp будут обязательно. Тут мы столкнемся с проблемой: Циан записывает float через запятую, Python - через точку, поэтому дефолтная float(str) не сработает. Набросаем свой конвертор."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 711,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def myStrToFloat(string):\n",
    "    delimiter = 0\n",
    "    value = 0\n",
    "    for i in range(0, len(string)):\n",
    "        if string[i] == ',':\n",
    "            delimiter = i\n",
    "    for i in range(0, delimiter):\n",
    "        value += int(string[delimiter - i - 1]) * 10 ** i\n",
    "\n",
    "    for i in range(1, len(string) - delimiter):\n",
    "        value += (int(string[delimiter + i]) * (10 ** (i - 2)))\n",
    "    return value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Totsp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 715,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getTotsp(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    space_block = re.split('Общая площадь', table)[1]\n",
    "   \n",
    "    total = re.split('Площадь комнат', space_block)[0]\n",
    "    total_space = re.split('\\n|\\xa0', total)[2]\n",
    "    if total_space.isdigit():\n",
    "        total_space = int(total_space)\n",
    "    else:\n",
    "        total_space = myStrToFloat(total_space)\n",
    "            \n",
    "    return total_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 724,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllTotsp(l, r):\n",
    "    \n",
    "    totsp = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        totsp.append(getTotsp(flat_page))\n",
    "        \n",
    "    return totsp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 718,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "totsp = getAllTotsp(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Livesp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 719,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getLivesp(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    space_block = re.split('Общая площадь', table)[1]\n",
    "   \n",
    "    living = re.split('Жилая площадь', space_block)[1]\n",
    "    living_space = re.split('\\n|\\xa0', living)[2]\n",
    "    if living_space.isdigit():\n",
    "        living_space = int(living_space)\n",
    "    else:\n",
    "        living_space = myStrToFloat(living_space)\n",
    "            \n",
    "    return living_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 722,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllLivesp(l, r):\n",
    "    \n",
    "    livesp = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        livesp.append(getLivesp(flat_page))\n",
    "        \n",
    "    return livesp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 725,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "livesp = getAllLivesp(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Переходим к опциональным полям"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kitsp, Bal, Tel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 726,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\nОбщая информация:\\n\\n\\nЭтаж:\\n\\n                3\\xa0/\\xa010\\n                \\n            \\n\\n\\nТип дома:\\n\\n            вторичка, \\n            кирпичный\\n        \\n\\n\\nТип продажи:\\nсвободная\\n\\n\\nОбщая площадь:\\n\\n34\\xa0м2\\n\\n\\n\\nПлощадь комнат:\\n\\n19\\xa0м2\\n\\n\\n\\nЖилая площадь:\\n\\n19\\xa0м2\\n\\n\\n\\nПлощадь кухни:\\n\\n6\\xa0м2\\n\\n\\n\\nСовмещенных санузлов:\\n1\\n            \\n\\n\\nБалкон:\\n1 балк.\\n        \\n\\n\\nЛифт:\\n1 пасс.\\n\\n\\nПарковка:\\nназемная\\n\\n\\nТелефон:\\nда\\n\\n\\nВид из окна:\\nдвор\\n\\n\\nРемонт:\\nкосметический\\n\\n'"
      ]
     },
     "execution_count": 726,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#flat_url = 'http://www.cian.ru/sale/flat/' + str(links[2]) + '/'\n",
    "flat_url = 'http://www.cian.ru/sale/flat/150387502/' \n",
    "flat_page = requests.get(flat_url)\n",
    "flat_page = flat_page.content\n",
    "flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "table = html_stripper(table)\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Нужный нам подблок информации мы снова можем ограничить только слева названием поля \"Общая площадь\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 727,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "':\\n\\n34\\xa0м2\\n\\n\\n\\nПлощадь комнат:\\n\\n19\\xa0м2\\n\\n\\n\\nЖилая площадь:\\n\\n19\\xa0м2\\n\\n\\n\\nПлощадь кухни:\\n\\n6\\xa0м2\\n\\n\\n\\nСовмещенных санузлов:\\n1\\n            \\n\\n\\nБалкон:\\n1 балк.\\n        \\n\\n\\nЛифт:\\n1 пасс.\\n\\n\\nПарковка:\\nназемная\\n\\n\\nТелефон:\\nда\\n\\n\\nВид из окна:\\nдвор\\n\\n\\nРемонт:\\nкосметический\\n\\n'"
      ]
     },
     "execution_count": 727,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "space_block = re.split('Общая площадь', table)[1]\n",
    "space_block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 728,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "':\\n\\n19\\xa0м2\\n\\n\\n\\nПлощадь кухни:\\n\\n6\\xa0м2\\n\\n\\n\\nСовмещенных санузлов:\\n1\\n            \\n\\n\\nБалкон:\\n1 балк.\\n        \\n\\n\\nЛифт:\\n1 пасс.\\n\\n\\nПарковка:\\nназемная\\n\\n\\nТелефон:\\nда\\n\\n\\nВид из окна:\\nдвор\\n\\n\\nРемонт:\\nкосметический\\n\\n'"
      ]
     },
     "execution_count": 728,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optional_block = re.split('Жилая площадь', space_block)[1]\n",
    "optional_block"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kitsp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Если площадь кухни не указана, то она фигурирует в таблице в виде прочерка. При этом она может быть float с запятой в качестве разделителя, как и предыдущие площади. Учтем это"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 729,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getKitsp(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    space_block = re.split('Общая площадь', table)[1]\n",
    "    optional_block = re.split('Жилая площадь', space_block)[1]\n",
    "    \n",
    "    kitchen_space = np.nan\n",
    "    \n",
    "    if 'Площадь кухни' in optional_block:\n",
    "        kitchen_block = re.split('Площадь кухни', optional_block)[1]\n",
    "        if re.split('\\n|\\xa0', kitchen_block)[2] != '–':\n",
    "            if re.split('\\n|\\xa0', kitchen_block)[2].isdigit():\n",
    "                kitchen_space = int(re.split('\\n|\\xa0', kitchen_block)[2])\n",
    "            else:\n",
    "                kitchen_space = myStrToFloat(re.split('\\n|\\xa0', kitchen_block)[2])\n",
    "            \n",
    "    return kitchen_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 730,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllKitsp(l, r):\n",
    "    \n",
    "    kitsp = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        kitsp.append(getKitsp(flat_page))\n",
    "        \n",
    "    return kitsp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 732,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "kitsp = getAllKitsp(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Балкона может не быть, балкон может быть, а еще балконов может быть больше одного. Информации о балконе может не быть.\n",
    "### Так что будем готовы к значениям > 1. Прочерк будет говорить о том, что информации нет, слово \"нет\" - о том, что есть информация о том, что нет балкона."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 737,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getBal(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    space_block = re.split('Общая площадь', table)[1]\n",
    "    optional_block = re.split('Жилая площадь', space_block)[1]\n",
    "    \n",
    "    balcony = np.nan\n",
    "    if 'Балкон' in optional_block:\n",
    "        balcony_block = re.split('Балкон', optional_block)[1]\n",
    "        if re.split('\\n', balcony_block)[1] != 'нет':\n",
    "            if re.split('\\n', balcony_block)[1] != '–':\n",
    "                balcony = int(re.split('\\n', balcony_block)[1][0])\n",
    "        else:\n",
    "            balcony = 0\n",
    "            \n",
    "    return balcony"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 738,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllBal(l, r):\n",
    "    \n",
    "    bal = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        bal.append(getBal(flat_page))\n",
    "        \n",
    "    return bal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 740,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bal = getAllBal(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Телефон может быть, может не быть, и о нем может не быть информации. О том, что он есть, нам сообщает слово \"да\", о том, что его нет - слово \"нет\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 741,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getTel(flat_page):\n",
    "    table = flat_page.find('table', attrs = {'class':'object_descr_props'})\n",
    "    table = html_stripper(table)\n",
    "    \n",
    "    space_block = re.split('Общая площадь', table)[1]\n",
    "    optional_block = re.split('Жилая площадь', space_block)[1]\n",
    "    \n",
    "    telephone = np.nan\n",
    "    if 'Телефон' in optional_block:\n",
    "        telephone_block = re.split('Телефон', optional_block)[1]\n",
    "        if re.split('\\n', telephone_block)[1] == 'да':\n",
    "            telephone = 1\n",
    "        elif re.split('\\n', telephone_block)[1] == 'нет':\n",
    "            telephone = 0\n",
    "            \n",
    "    return telephone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 742,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getAllTel(l, r):\n",
    "    \n",
    "    tel = []\n",
    "    \n",
    "    for i in range(l, r):\n",
    "        flat_url = 'http://www.cian.ru/sale/flat/' + str(links[i]) + '/'\n",
    "        flat_page = requests.get(flat_url)\n",
    "        flat_page = flat_page.content\n",
    "        flat_page = BeautifulSoup(flat_page, 'lxml')\n",
    "        tel.append(getTel(flat_page))\n",
    "        \n",
    "    return tel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 744,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tel = getAllTel(0, len(links))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Последние штрихи перед сборкой DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 745,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = []\n",
    "for i in range(0, len(links)):\n",
    "    N.append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### В этом ноутбуке квартиры только из ЦАО, укажу это"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# District"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 746,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "district = []\n",
    "for i in range(0, len(links)):\n",
    "    district.append('CAD')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Собираем все воедино"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 808,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = dict([('New', new), ('Bal', bal), ('Tel', tel), ('Walk', walk), ('Metrdist', metrdist), ('Nfloors', nfloors), ('Floor', floor), ('Totsp', totsp), ('Livesp', livesp), ('Kitsp', kitsp), ('N', N), ('Price', prices), ('Rooms', rooms), ('Distance', distance), ('Brick', bricks), ('District', district)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 809,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 810,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>802</th>\n",
       "      <th>803</th>\n",
       "      <th>804</th>\n",
       "      <th>805</th>\n",
       "      <th>806</th>\n",
       "      <th>807</th>\n",
       "      <th>808</th>\n",
       "      <th>809</th>\n",
       "      <th>810</th>\n",
       "      <th>811</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Bal</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brick</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Distance</th>\n",
       "      <td>1.45216</td>\n",
       "      <td>1.73696</td>\n",
       "      <td>1.75092</td>\n",
       "      <td>2.71596</td>\n",
       "      <td>4.19487</td>\n",
       "      <td>4.05345</td>\n",
       "      <td>4.05345</td>\n",
       "      <td>4.09831</td>\n",
       "      <td>4.05345</td>\n",
       "      <td>4.50236</td>\n",
       "      <td>...</td>\n",
       "      <td>4.49908</td>\n",
       "      <td>3.05</td>\n",
       "      <td>4.06076</td>\n",
       "      <td>4.12913</td>\n",
       "      <td>3.27558</td>\n",
       "      <td>2.04415</td>\n",
       "      <td>5.22719</td>\n",
       "      <td>2.09632</td>\n",
       "      <td>2.19363</td>\n",
       "      <td>1.54887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>District</th>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>...</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "      <td>CAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Floor</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kitsp</th>\n",
       "      <td>15</td>\n",
       "      <td>12</td>\n",
       "      <td>7</td>\n",
       "      <td>40</td>\n",
       "      <td>15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>6.1</td>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Livesp</th>\n",
       "      <td>89</td>\n",
       "      <td>94</td>\n",
       "      <td>30</td>\n",
       "      <td>162</td>\n",
       "      <td>35</td>\n",
       "      <td>22.7</td>\n",
       "      <td>17.3</td>\n",
       "      <td>17.3</td>\n",
       "      <td>10</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>42.8</td>\n",
       "      <td>37</td>\n",
       "      <td>60.9</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>51</td>\n",
       "      <td>35</td>\n",
       "      <td>47</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Metrdist</th>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>802</td>\n",
       "      <td>803</td>\n",
       "      <td>804</td>\n",
       "      <td>805</td>\n",
       "      <td>806</td>\n",
       "      <td>807</td>\n",
       "      <td>808</td>\n",
       "      <td>809</td>\n",
       "      <td>810</td>\n",
       "      <td>811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nfloors</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>14</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Price</th>\n",
       "      <td>55000000</td>\n",
       "      <td>55000000</td>\n",
       "      <td>15400000</td>\n",
       "      <td>2500000</td>\n",
       "      <td>3750000</td>\n",
       "      <td>4403057</td>\n",
       "      <td>4585315</td>\n",
       "      <td>4677021</td>\n",
       "      <td>4814581</td>\n",
       "      <td>5100000</td>\n",
       "      <td>...</td>\n",
       "      <td>19500000</td>\n",
       "      <td>19500000</td>\n",
       "      <td>19561000</td>\n",
       "      <td>19610000</td>\n",
       "      <td>19723264</td>\n",
       "      <td>19800000</td>\n",
       "      <td>19800000</td>\n",
       "      <td>19900000</td>\n",
       "      <td>19900000</td>\n",
       "      <td>19900000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rooms</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tel</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Totsp</th>\n",
       "      <td>117</td>\n",
       "      <td>145</td>\n",
       "      <td>46</td>\n",
       "      <td>232</td>\n",
       "      <td>70</td>\n",
       "      <td>22.7</td>\n",
       "      <td>17.3</td>\n",
       "      <td>17.3</td>\n",
       "      <td>17.3</td>\n",
       "      <td>32</td>\n",
       "      <td>...</td>\n",
       "      <td>65.4</td>\n",
       "      <td>70</td>\n",
       "      <td>60.9</td>\n",
       "      <td>85</td>\n",
       "      <td>54.4</td>\n",
       "      <td>50.8</td>\n",
       "      <td>82</td>\n",
       "      <td>56</td>\n",
       "      <td>68</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Walk</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16 rows × 812 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               0         1         2        3        4        5        6    \\\n",
       "Bal              0       NaN         1      NaN      NaN        0        0   \n",
       "Brick          NaN         1         1        1        1      NaN      NaN   \n",
       "Distance   1.45216   1.73696   1.75092  2.71596  4.19487  4.05345  4.05345   \n",
       "District       CAD       CAD       CAD      CAD      CAD      CAD      CAD   \n",
       "Floor            2         2         3        2        9        3        2   \n",
       "Kitsp           15        12         7       40       15      NaN      NaN   \n",
       "Livesp          89        94        30      162       35     22.7     17.3   \n",
       "Metrdist        11        10         3        7        5        6        6   \n",
       "N                0         1         2        3        4        5        6   \n",
       "New              0         0         0        0        0        1        1   \n",
       "Nfloors          4         7         8       14       11      NaN      NaN   \n",
       "Price     55000000  55000000  15400000  2500000  3750000  4403057  4585315   \n",
       "Rooms            4         5         2        5        2        1        1   \n",
       "Tel              1         1       NaN        0      NaN      NaN      NaN   \n",
       "Totsp          117       145        46      232       70     22.7     17.3   \n",
       "Walk             1         1         1        1        1        1        1   \n",
       "\n",
       "              7        8        9      ...          802       803       804  \\\n",
       "Bal             0      NaN      NaN    ...            1       NaN         1   \n",
       "Brick           1        1      NaN    ...            1         0         1   \n",
       "Distance  4.09831  4.05345  4.50236    ...      4.49908      3.05   4.06076   \n",
       "District      CAD      CAD      CAD    ...          CAD       CAD       CAD   \n",
       "Floor           2        2        1    ...            5        10         3   \n",
       "Kitsp         NaN      NaN        7    ...          6.1        12       NaN   \n",
       "Livesp       17.3       10       20    ...         42.8        37      60.9   \n",
       "Metrdist        5        6        3    ...            8         5         7   \n",
       "N               7        8        9    ...          802       803       804   \n",
       "New             0        1        0    ...            0         0         0   \n",
       "Nfloors         6        5        9    ...            8        10         5   \n",
       "Price     4677021  4814581  5100000    ...     19500000  19500000  19561000   \n",
       "Rooms           1        1        1    ...            3         2         2   \n",
       "Tel             0      NaN        1    ...            1         1         0   \n",
       "Totsp        17.3     17.3       32    ...         65.4        70      60.9   \n",
       "Walk            1        1        1    ...            1         1         1   \n",
       "\n",
       "               805       806       807       808       809       810       811  \n",
       "Bal              1       NaN         1         1         1         0       NaN  \n",
       "Brick            1       NaN         1       NaN         0         0         0  \n",
       "Distance   4.12913   3.27558   2.04415   5.22719   2.09632   2.19363   1.54887  \n",
       "District       CAD       CAD       CAD       CAD       CAD       CAD       CAD  \n",
       "Floor            3         2         3         7         5         6         1  \n",
       "Kitsp           12       NaN        20        10         8         9         6  \n",
       "Livesp          54         0        30        51        35        47        37  \n",
       "Metrdist         5        13        12        15         4         2         4  \n",
       "N              805       806       807       808       809       810       811  \n",
       "New              0         1         1         0         0         0         0  \n",
       "Nfloors          5         7         8        14         9        14         6  \n",
       "Price     19610000  19723264  19800000  19800000  19900000  19900000  19900000  \n",
       "Rooms            3         2         1         3         2         3         2  \n",
       "Tel            NaN       NaN       NaN       NaN         1         1         1  \n",
       "Totsp           85      54.4      50.8        82        56        68        58  \n",
       "Walk             1         1         1         1         1         0         1  \n",
       "\n",
       "[16 rows x 812 columns]"
      ]
     },
     "execution_count": 810,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 811,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.to_csv('cian.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Остальные округи буду считать этим же ноутбуком, так что по содержательной части всё."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}