parse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from lxml import html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def text_or_none(tag):\n",
    "    if len(tag):\n",
    "        if tag[0] is not None:\n",
    "            return tag[0].text.rstrip().lstrip()\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "def parse_position(page):\n",
    "\n",
    "    tree = html.fromstring(page)\n",
    "\n",
    "    product_title = text_or_none(tree.xpath('//h1[@class=\"title\"]'))\n",
    "    product_subtitle = text_or_none(tree.xpath('//h2[@class=\"subtitle\"]'))\n",
    "\n",
    "    for tag in tree.xpath('//li'):\n",
    "        data = tag.xpath('div[@class=\"data\"]')\n",
    "        title = tag.xpath('div[@class=\"title\"]')\n",
    "        if not title:\n",
    "            continue\n",
    "        else:\n",
    "            title = title[0]\n",
    "            av_storage = \\\n",
    "                text_or_none(title.xpath('div[@class=\"av_storage\"]/span'))\n",
    "            text_muted_subregion = \\\n",
    "                text_or_none(title.xpath('div[@class=\"text-muted subregion\"]'))\n",
    "            title_full_text = title.text_content().replace('\\n', '').rstrip().lstrip()\n",
    "        if not data:\n",
    "            continue\n",
    "        else:\n",
    "            data = data[0]\n",
    "            weight = \\\n",
    "                text_or_none(data.xpath('div[@class=\"av_nal\"]/span')).replace('г', '')\n",
    "            price_btc = \\\n",
    "                text_or_none(data.xpath('div[@class=\"av_price\"]/span'))\n",
    "            price = \\\n",
    "                text_or_none(data.xpath('div[@class=\"av_price\"]/b'))\\\n",
    "                    .replace('руб', '')\n",
    "        product = (product_title,\n",
    "                   product_subtitle,\n",
    "                   title_full_text,\n",
    "                   text_muted_subregion,\n",
    "                   av_storage,\n",
    "                   weight,\n",
    "                   price,\n",
    "                   price_btc)\n",
    "        return product\n",
    "    \n",
    "def parse_comments(tree_comment):\n",
    "    page_comments = list()\n",
    "    comments_xpath = '//li/div[@class=\"comment\"]'\n",
    "    deals_count_xpath = 'a/div[@class=\"review-user-data text-muted\"]'\n",
    "    user_name_xpath = 'div[@class=\"comment_cont\"]/div[@class=\"comment_top\"]/a/b'\n",
    "    comment_date_xpath = 'div[@class=\"comment_cont\"]/div[@class=\"comment_top\"]\\\n",
    "        /span[@class=\"date text-muted\"]'\n",
    "    comment_product_xpath = 'div[@class=\"comment_cont\"]/div[@class=\"comment_top\"]\\\n",
    "        /div[@class=\"pull-right\"]/a'\n",
    "    comment_text_xpath = 'div[@class=\"comment_cont\"]/div[@class=\"text clearfix\"]'\n",
    "    comment_rate_xpath = 'div[@class=\"comment_cont\"]/div[@class=\"comment_foot\"]\\\n",
    "        /div[@class=\"rating_star\"]'\n",
    "    comment_city_xpath = 'div[@class=\"comment_cont\"]/div[@class=\"comment_foot\"]\\\n",
    "        /i[@class=\"text-muted pull-right\"]'\n",
    "    user_link_xpath = 'a[@class=\"cover avatar\"]'\n",
    "    comments = tree_comment.xpath(comments_xpath)\n",
    "\n",
    "    for tagzor in comments:\n",
    "        try:\n",
    "            comment_parsed = dict()\n",
    "            comment_parsed['deals_count'] = \\\n",
    "                text_or_none(tagzor.xpath(deals_count_xpath))\n",
    "            if comment_parsed['deals_count']:\n",
    "                comment_parsed['deals_count'] = \\\n",
    "                    comment_parsed['deals_count'].split(': ')[1]\n",
    "            comment_parsed['user_name'] = \\\n",
    "                text_or_none(tagzor.xpath(user_name_xpath))\n",
    "            comment_parsed['comment_date'] = \\\n",
    "                text_or_none(tagzor.xpath(comment_date_xpath))\n",
    "            comment_parsed['comment_text'] = \\\n",
    "                text_or_none(tagzor.xpath(comment_text_xpath))\n",
    "            comment_parsed['comment_city'] = \\\n",
    "                text_or_none(tagzor.xpath(comment_city_xpath))\n",
    "            comment_parsed['user_link'] = \\\n",
    "                tagzor.xpath(user_link_xpath)[0].get('href')\n",
    "            comment_parsed['comment_product_link'] = \\\n",
    "                tagzor.xpath(comment_product_xpath)\n",
    "\n",
    "            if comment_parsed['comment_product_link']:\n",
    "                comment_parsed['comment_product_link'] = \\\n",
    "                    comment_parsed['comment_product_link'][0].get('href')\n",
    "                comment_parsed['user_type'] = 'buyer'\n",
    "            else:\n",
    "                comment_parsed['comment_product_link'] = None\n",
    "                comment_parsed['user_type'] = 'seller'\n",
    "            comment_parsed['comment_product_text'] = \\\n",
    "                text_or_none(tagzor.xpath(comment_product_xpath))\n",
    "            comment_parsed['comment_rate'] = \\\n",
    "                tagzor.xpath(comment_rate_xpath)\n",
    "\n",
    "            if comment_parsed['comment_rate']:\n",
    "                comment_parsed['comment_rate'] = \\\n",
    "                    comment_parsed['comment_rate'][0].get('href')\n",
    "                comment_parsed['user_type'] = 'buyer'\n",
    "            else:\n",
    "                comment_parsed['comment_rate'] = None\n",
    "                comment_parsed['user_type'] = 'seller'\n",
    "\n",
    "            yield comment_parsed\n",
    "        except:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_star(tag):\n",
    "    tag_href = tag.get('href')\n",
    "    tag_text = tag.text\n",
    "    if '%' in tag_text:\n",
    "        return None\n",
    "    rating = tag_href.split('rate=')[1]\n",
    "    counter = tag_text.split(' ')[0]\n",
    "    return (rating, counter)\n",
    "\n",
    "def parse_market(tree_page):\n",
    "    market_parsed = dict()\n",
    "    shop_deals_count_xpath = '//div[@class=\"header_shop__info\"]/ul[@class=\"text-muted\"]/li/a'\n",
    "    shop_header_xpath = '//div[@class=\"header_shop__info\"]/h1[@class=\"title over\"]'\n",
    "    review_rate_xpath = '//div[@class=\"col-md-3 col-sm-4\"]'\n",
    "    review_rate_mark_xpath = \\\n",
    "        '//div[@class=\"col-md-3 col-sm-4\"]/div[@class=\"review-rate-filter\"]/small/a'\n",
    "    \n",
    "    for rate_mark in tree_page.xpath(review_rate_mark_xpath):\n",
    "        rate = parse_star(rate_mark)\n",
    "        if rate:\n",
    "            market_parsed[rate[0]] = rate[1]\n",
    "    market_parsed['shop_header'] = \\\n",
    "        text_or_none(tree_page.xpath(shop_header_xpath))\n",
    "#     market_parsed['shop_link'] = \\\n",
    "#         tree_page.xpath(shop_deals_count_xpath)[0].get('href')\n",
    "    market_parsed['shop_deals_count'] = \\\n",
    "        text_or_none(tree_page.xpath(shop_deals_count_xpath))\n",
    "    return market_parsed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "product_categories = [\n",
    "    'Хакинг',\n",
    "    'Цифровые товары',\n",
    "    'Семена',\n",
    "    'Аптека',\n",
    "    'Эйфоретики',\n",
    "    'Бошка',\n",
    "    'Кокаин',\n",
    "    'Гашиш',\n",
    "    'Базы',\n",
    "    'SSH, VPN',\n",
    "    'Альфа-ПВП',\n",
    "    'Мефедрон',\n",
    "    'Сканы',\n",
    "    'Приборы и оборудование',\n",
    "    'Обнал BTC',\n",
    "    'Масло/Экстракты', \n",
    "    'Амфетамин', \n",
    "    'MDMA', \n",
    "    'Банковские карты',\n",
    "    'Документы', \n",
    "    'Кошельки', \n",
    "    'Разное', \n",
    "    'Пробив', \n",
    "    'Экстази', \n",
    "    'LSD',\n",
    "    'Автодокументы', \n",
    "    'Cтимуляторы', \n",
    "    'Химические реактивы', \n",
    "    'Марихуана',\n",
    "    'Дизайн и графика', \n",
    "    'Метадон', \n",
    "    'Триптамины', \n",
    "    'Другое',\n",
    "    'Диссоциативы', \n",
    "    'Наружная реклама', \n",
    "    'Анаболики/Стероиды',\n",
    "    'Химические реактивы/Конструкторы', \n",
    "    'Конструкторы', \n",
    "    'План',\n",
    "    'Метамфетамин', \n",
    "    'Героин', \n",
    "    'Грибы', \n",
    "    'SIM-карты', \n",
    "    'Карты, SIM',\n",
    "    'CannaFood', \n",
    "    'Реагент', \n",
    "    'DMT', \n",
    "    'Удостоверения', \n",
    "    'Бутираты', \n",
    "    '2С-*',\n",
    "    'Фальшивые деньги', \n",
    "    'NBOME', \n",
    "    'DO*', \n",
    "    'Твердый Микс', \n",
    "    'Каннабиноиды',\n",
    "    '1-фенил-2-нитропропен', \n",
    "    'Мескалин', \n",
    "    'Россыпь/Микс', \n",
    "    'MDA', \n",
    "    'MDPV',\n",
    "    'Энтеогены', \n",
    "    'Основы для смесей', \n",
    "    'Опиаты', \n",
    "    'Катиноны',\n",
    "    'Психоделики', \n",
    "    'Эфедрон', \n",
    "    'Винт', \n",
    "    'Майнинг криптовалют'\n",
    "]\n",
    "\n",
    "drugs = [\n",
    "        'Аптека', 'Эйфоретики','Бошка', 'Кокаин', 'Гашиш', 'Альфа-ПВП',\n",
    "        'Мефедрон', 'Масло/Экстракты', 'Амфетамин', 'MDMA', 'Экстази', 'LSD',\n",
    "        'Cтимуляторы', 'Марихуана','Метадон', 'Триптамины',\n",
    "        'Диссоциативы', 'Анаболики/Стероиды', 'План',\n",
    "        'Метамфетамин', 'Героин', 'Грибы', 'Мука', 'Кристалл',\n",
    "        'CannaFood', 'Реагент', 'DMT', 'Бутираты', '2С-*',\n",
    "        'NBOME', 'DO*', 'Твердый Микс', 'Каннабиноиды',\n",
    "        '1-фенил-2-нитропропен', 'Мескалин', 'Россыпь/Микс', 'MDA', 'MDPV',\n",
    "        'Энтеогены', 'Опиаты', 'Катиноны',\n",
    "        'Психоделики', 'Эфедрон', 'Винт'\n",
    "        ]\n",
    "\n",
    "jobs = [\n",
    "    'Партнёрство и Франшиза',\n",
    "    'Работа',\n",
    "    'Курьер/Перевозчик',\n",
    "    'Другая работа',\n",
    "    'Трафаретчик',\n",
    "    'Кладмен',\n",
    "    'Наружная реклама',\n",
    "    'Склад',\n",
    "    'Дизайн и графика',\n",
    "    'Промоутер'\n",
    "]\n",
    "\n",
    "production = [\n",
    "    'Реагент',\n",
    "    'Химические реактивы/Конструкторы', \n",
    "    'Конструкторы',\n",
    "    'Приборы и оборудование',\n",
    "    'Химические реактивы',\n",
    "    'Семена'\n",
    "]\n",
    "\n",
    "operations = [\n",
    "    'Обнал BTC',\n",
    "    'Дизайн и графика',\n",
    "    'Наружная реклама',\n",
    "    'Майнинг криптовалют',\n",
    "    'Банковские карты',\n",
    "    'SIM-карты', \n",
    "    'Карты, SIM',\n",
    "    'Фальшивые деньги',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = 'catalog/page/product'\n",
    "\n",
    "all_products = list()\n",
    "errors = 0\n",
    "fieldnames = ['Название позиции',\n",
    "              'Тип позиции',\n",
    "              'Полное описание',\n",
    "              'Уточненная локация',\n",
    "              'Тип закладки',\n",
    "              'Масса',\n",
    "              'Стоимость',\n",
    "              'price_btc']\n",
    "\n",
    "for filename in os.listdir(directory):\n",
    "    filepath = directory + '/' + filename\n",
    "    try:\n",
    "        with open(filepath) as f:\n",
    "            read_data = f.read()\n",
    "            product = parse_position(read_data)\n",
    "    except:\n",
    "        errors += 1\n",
    "        continue\n",
    "    if product:\n",
    "        product_parsed = dict(zip(fieldnames, product))\n",
    "        product_parsed['id'] = filename\n",
    "        all_products.append(product_parsed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "# можно засунуть внутрь цикла выше\n",
    "# все вытащить, потом обрабатывать — надежнее\n",
    "\n",
    "for product in all_products:\n",
    "    product['title_full_text_replaced'] = product['Полное описание']\\\n",
    "        .replace('Прикоп', '')\\\n",
    "        .replace('Тайник', '')\\\n",
    "        .replace('Магнит', '')\\\n",
    "        .replace(', ', '')\\\n",
    "        .replace(',','')\\\n",
    "        .replace('\\\n",
    "        ','')\\\n",
    "        .rstrip()\\\n",
    "        .lstrip()\\\n",
    "        .split(':')[0]\n",
    "    product['Стоимость'] = product['Стоимость']\\\n",
    "                            .replace(' ', '')\n",
    "    if 'шт' in product['Масса']:\n",
    "        product['Мера'] = 'штуки'\n",
    "        product['Масса'] = product['Масса'].replace('\\xa0шт', '').replace('\\xa0', '')\n",
    "    elif 'мл' in product['Масса']:\n",
    "        product['Мера'] = 'мл'\n",
    "        product['Масса'] = product['Масса'].replace('мл', '').replace('\\xa0', '')\n",
    "    elif 'м' in product['Масса']:\n",
    "        product['Мера'] = 'м'\n",
    "        product['Масса'] = product['Масса'].replace('м', '').replace('\\xa0', '')\n",
    "    else:\n",
    "        product['Мера'] = 'граммы'\n",
    "        product['Масса'] = product['Масса'].replace('\\xa0', '')\n",
    "    if '$' in product['Стоимость']:\n",
    "        product['Валюта'] = 'доллар'\n",
    "        product['Стоимость'] = product['Стоимость'].replace('$', '')\n",
    "    else:\n",
    "        product['Валюта'] = 'рубль'\n",
    "    product['Тип позиции'] = product['Тип позиции'].replace('Кристалл', 'Мефедрон')\n",
    "    product['Тип позиции'] = product['Тип позиции'].replace('Бошка', 'Марихуана')\n",
    "    if 'PVP' not in product['Название позиции']:\n",
    "        product['Тип позиции'] = product['Тип позиции'].replace('Мука', 'Мефедрон')\n",
    "    elif 'PVP' in product['Название позиции']:\n",
    "        product['Тип позиции'] = 'Альфа-ПВП'\n",
    "    if '(' in product['title_full_text_replaced']:\n",
    "        product['title_full_text_replaced'] = \\\n",
    "            product['title_full_text_replaced'].split(' (')[0]\n",
    "df = pd.DataFrame(all_products)\n",
    "df.drop('price_btc', axis=1, inplace=True)\n",
    "df = df.astype({'Стоимость': 'int64', 'Масса': 'float'})\n",
    "df['Город'] = df['title_full_text_replaced']\n",
    "df.drop('title_full_text_replaced', inplace=True, axis=1)\n",
    "df.drop(2606, inplace=True)\n",
    "\n",
    "cities_df = pd.read_csv('./cities.csv')\n",
    "cities_df.drop(['Индекс', 'Тип города', 'Тип региона', 'Тип района', 'Район',\n",
    "                'Тип н/п', 'Н/п', 'Код КЛАДР', 'Код ФИАС', 'Уровень по ФИАС',\n",
    "                'Признак центра района или региона', 'Код ОКАТО', 'Код ОКТМО',\n",
    "                'Код ИФНС'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_drug = df[df['Тип позиции'].isin(drugs)]\n",
    "# df_drug = df_merged[df_merged['Тип позиции'].isin(drugs)]\n",
    "group_by_type = \\\n",
    "    df_drug\\\n",
    "        [df_drug['Мера'] == 'граммы']\\\n",
    "        [df_drug['Масса'] < 1000]\\\n",
    "        .groupby('Тип позиции')\\\n",
    "        .sum()\n",
    "\n",
    "group_by_type_count = \\\n",
    "    df_drug\\\n",
    "        [df_drug['Мера'] == 'граммы']\\\n",
    "        [df_drug['Масса'] < 1000]\\\n",
    "        .groupby('Тип позиции')\\\n",
    "        .count()\n",
    "\n",
    "group_by_type['Масса кг'] = group_by_type['Масса']/1000\n",
    "group_by_type.drop('Масса', axis=1, inplace=True)\n",
    "\n",
    "group_by_type['Количество предложений в день'] = group_by_type_count['id']\n",
    "\n",
    "group_by_type['Месячная стоимость'] = group_by_type['Стоимость']*26\n",
    "\n",
    "group_by_type['Месячная стоимость в панамерах'] = \\\n",
    "    group_by_type['Месячная стоимость']/15000000\n",
    "group_by_type['Месячная стоимость в айфонах'] = \\\n",
    "    group_by_type['Месячная стоимость']/80000\n",
    "\n",
    "group_by_type.sort_values('Стоимость', ascending=False)\n",
    "group_by_type.to_csv('group_by_type.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_by_city = \\\n",
    "    df_drug\\\n",
    "        [df_drug['Мера'] == 'граммы']\\\n",
    "        [df_drug['Масса'] < 1000]\\\n",
    "        .groupby(['Город'])\\\n",
    "        .sum()\n",
    "\n",
    "drug_by_city_count = \\\n",
    "    df_drug\\\n",
    "        [df_drug['Мера'] == 'граммы']\\\n",
    "        [df_drug['Масса'] < 1000]\\\n",
    "        .groupby(['Город'])\\\n",
    "        .count()\n",
    "\n",
    "group_by_city['Масса кг'] = group_by_city['Масса']/1000\n",
    "group_by_city.drop('Масса', axis=1, inplace=True)\n",
    "\n",
    "group_by_city['Количество предложений в день'] = drug_by_city_count['id']\n",
    "\n",
    "group_by_city['Месячная стоимость'] = group_by_city['Стоимость']*26\n",
    "\n",
    "group_by_city['Месячная стоимость в панамерах'] = \\\n",
    "    group_by_city['Месячная стоимость']/15000000\n",
    "group_by_city['Месячная стоимость в айфонах'] = \\\n",
    "    group_by_city['Месячная стоимость']/80000\n",
    "\n",
    "group_by_city.sort_values('Стоимость', ascending=False)\n",
    "group_by_city.to_csv('group_by_city.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_by_city_type = \\\n",
    "    df_drug\\\n",
    "        [df_drug['Мера'] == 'граммы']\\\n",
    "        [df_drug['Масса'] < 1000]\\\n",
    "        .groupby(['Город','Тип позиции'])\\\n",
    "        .sum()\n",
    "\n",
    "group_by_city_type_count = \\\n",
    "    df_drug\\\n",
    "        [df_drug['Мера'] == 'граммы']\\\n",
    "        [df_drug['Масса'] < 1000]\\\n",
    "        .groupby(['Город','Тип позиции'])\\\n",
    "        .count()\n",
    "\n",
    "group_by_city_type['Масса кг'] = group_by_city_type['Масса']/1000\n",
    "group_by_city_type.drop('Масса', axis=1, inplace=True)\n",
    "\n",
    "group_by_city_type['Количество предложений в день'] = group_by_city_type_count['id']\n",
    "\n",
    "group_by_city_type.to_csv('group_by_city_type.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_comments = list()\n",
    "files_done = list()\n",
    "i = 0\n",
    "comments_market_dir = \\\n",
    "    'catalog/page/expiriences/market'\n",
    "for market in os.listdir(comments_market_dir):\n",
    "    market_products = comments_market_dir + '/' + market\n",
    "    for product in os.listdir(market_products):\n",
    "        product_pages = market_products + '/' + product\n",
    "        pid = product.split('product_id=')[1]\n",
    "        for page in os.listdir(product_pages):\n",
    "            filepath = product_pages + '/' + page\n",
    "            if i%10000 == 0:\n",
    "                print(i)\n",
    "            with open(filepath) as f:\n",
    "                try:\n",
    "                    read_data = f.read()\n",
    "                    tree_comment = html.fromstring(read_data)\n",
    "                except:\n",
    "                    continue\n",
    "            for comment in parse_comments(tree_comment):\n",
    "                comment['market'] = market\n",
    "                comment['pid'] = pid\n",
    "                comment['page'] = page\n",
    "                all_comments.append(comment)\n",
    "            files_done.append(filepath)\n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# можно засунуть внутрь цикла выше\n",
    "# все вытащить, потом обрабатывать — надежнее\n",
    "\n",
    "for comment in all_comments:\n",
    "    date, time = comment['comment_date'].split(\" в \")\n",
    "    if 'сегодня' in date:\n",
    "        comment['date'] = '05.08.2019'\n",
    "        comment['time'] = time\n",
    "        continue\n",
    "    if 'вчера' in date:\n",
    "        comment['date'] = '04.08.2019'\n",
    "        comment['time'] = time\n",
    "        continue\n",
    "    if '2018' not in date and '2017' not in date:\n",
    "        date = date + ' 2019'\n",
    "    date = date\\\n",
    "        .replace(' Января ','.01.')\\\n",
    "        .replace(' Февраля ','.02.')\\\n",
    "        .replace(' Марта ','.03.')\\\n",
    "        .replace(' Апреля ','.04.')\\\n",
    "        .replace(' Мая ','.05.')\\\n",
    "        .replace(' Июня ','.06.')\\\n",
    "        .replace(' Июля ','.07.')\\\n",
    "        .replace(' Августа ','.08.')\\\n",
    "        .replace(' Сентября ','.09.')\\\n",
    "        .replace(' Октября ','.10.')\\\n",
    "        .replace(' Ноября ','.11.')\\\n",
    "        .replace(' Декабря ','.12.')\n",
    "    comment['date'] = date\n",
    "    comment['time'] = time\n",
    "\n",
    "dfc = pd.DataFrame(all_comments)\n",
    "dfc.to_csv('comments_fixed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc['pub_date'] = pd.to_datetime(dfc['date'])\n",
    "dfc['date'] = dfc['pub_date']\n",
    "\n",
    "dfc.drop('pub_date', axis=1, inplace=True)\n",
    "\n",
    "dfct = dfc.set_index('date')\n",
    "\n",
    "df_month = \\\n",
    "    dfct[dfct['user_type']== 'buyer'][dfct['pub_date'] < datetime.date(2019, 9, 1)]\\\n",
    "    .resample('M').count()\n",
    "df_month[:'2019-8-1 00:00:00']['user_name'].to_csv('reviews.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_markets = list()\n",
    "markets_dir = 'catalog/page/markets'\n",
    "for market in os.listdir(markets_dir):\n",
    "    market_path = markets_dir + '/' + market\n",
    "    with open(market_path) as f:\n",
    "        try:\n",
    "            read_data = f.read()\n",
    "            tree = html.fromstring(read_data)\n",
    "        except:\n",
    "            continue\n",
    "    market_parsed = parse_market(tree)\n",
    "    all_markets.append(market_parsed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for market in all_markets:\n",
    "    if market['shop_deals_count']:\n",
    "        if '+' in market['shop_deals_count']:\n",
    "            market['shop_deals_count_true'] = market['shop_deals_count'].split('+')[0]\n",
    "        elif 'сдел' in market['shop_deals_count']:\n",
    "            market['shop_deals_count_true'] = \\\n",
    "                market['shop_deals_count']\\\n",
    "                    .replace('сделок', '')\\\n",
    "                    .replace('сделки', '')\\\n",
    "                    .replace('сделка', '')\n",
    "    else:\n",
    "        market['shop_deals_count_true'] = 0\n",
    "dfm = pd.DataFrame(all_markets)\n",
    "dfm.drop('shop_deals_count', axis=1, inplace=True)\n",
    "dfm.drop(966, inplace=True)\n",
    "\n",
    "dfm['deals'] = dfm['shop_deals_count_true'].astype('int64')\n",
    "\n",
    "dfm1 = dfm.dropna(subset=['0']).astype({\n",
    "    '10':'int64',\n",
    "    '8':'int64',\n",
    "    '6':'int64',\n",
    "    '4':'int64',\n",
    "    '2':'int64',\n",
    "    '0':'int64',\n",
    "    'shop_deals_count_true':'int64'\n",
    "})\n",
    "\n",
    "dfm1['summary_comments'] = \\\n",
    "    dfm1['10'] + dfm1['8'] + dfm1['6'] + dfm1['4'] + dfm1['2'] + dfm1['0']\n",
    "dfm1['commented_share'] = \\\n",
    "    dfm1['summary_comments']/dfm1['shop_deals_count_true'] * 100\n",
    "\n",
    "dfm1 = dfm1[dfm1['commented_share'] < 100]\n",
    "dfm1.to_csv('./markets_full.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "users_dir = 'user'\n",
    "\n",
    "all_users = list()\n",
    "\n",
    "for filename in os.listdir(users_dir):\n",
    "    filepath = users_dir + '/' + filename\n",
    "    try:\n",
    "        with open(filepath) as f:\n",
    "            read_data = f.read()\n",
    "        tree = html.fromstring(read_data)\n",
    "        user = dict()\n",
    "\n",
    "        user['username'] = \\\n",
    "            text_or_none(tree.xpath('//div[@class=\"title h1 over\"]'))\n",
    "        user['regdate'] = \\\n",
    "            text_or_none(tree.xpath('//div[@class=\"col-xs-12 col-md-5\"]/div/b'))\n",
    "        all_users.append(user)\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "udf = pd.DataFrame(all_users)\n",
    "udf['date'] = pd.to_datetime(udf['regdate'])\n",
    "udf.drop('regdate', axis=1, inplace=True)\n",
    "udf = udf.set_index('date')\n",
    "\n",
    "udf_month = udf.resample('M').count()\n",
    "udf_month[:'2019-8-1 00:00:00'].to_csv('new_users_by_month.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = 'catalog/page/product'\n",
    "\n",
    "all_products = list()\n",
    "errors = 0\n",
    "fieldnames = ['Название позиции',\n",
    "              'Тип позиции',\n",
    "              'Полное описание',\n",
    "              'Уточненная локация',\n",
    "              'Тип закладки',\n",
    "              'Масса',\n",
    "              'Стоимость',\n",
    "              'price_btc']\n",
    "\n",
    "for filename in os.listdir(directory):\n",
    "    filepath = directory + '/' + filename\n",
    "    try:\n",
    "        with open(filepath) as f:\n",
    "            read_data = f.read()\n",
    "            product = parse_position(read_data)\n",
    "    except:\n",
    "        errors += 1\n",
    "        continue\n",
    "    if product:\n",
    "        product_parsed = dict(zip(fieldnames, product))\n",
    "        product_parsed['id'] = filename\n",
    "        all_products.append(product_parsed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}