Parsemachine lesson4

parsing/4/bot.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+import datetime
+import os
+
+from telegram import InlineKeyboardButton, InlineKeyboardMarkup
+from telegram.ext import Updater, CommandHandler, CallbackQueryHandler
+
+from main import OUT_FILENAME, OUT_XLSX_FILENAME
+
+
+# TOKEN Telegram-бота, получить можно у @BotFather
+TELEGRAM_TOKEN = ''
+
+
+def start_handler(update, context):
+    text = 'Привет! Выбери, в каком формате прислать парсинг?'
+    keyboard = [
+        [InlineKeyboardButton(text='Получить JSON', callback_data='get_json'), ],
+        [InlineKeyboardButton(text='Получить XLSX', callback_data='get_xlsx'), ],
+    ]
+    markup = InlineKeyboardMarkup(keyboard)
+    update.message.reply_text(text=text, reply_markup=markup)
+
+
+def callback_handler(update, context):
+    query = update.callback_query
+    query.answer()
+    filenames = {
+        'get_json': OUT_FILENAME,
+        'get_xlsx': OUT_XLSX_FILENAME,
+    }
+    filename = filenames.get(query.data)
+    if filename:
+        modified_at = datetime.datetime.fromtimestamp(os.path.getmtime(filename)).strftime('%Y-%m-%d %H:%M:%S')
+        caption = 'Результат парсинга от {}.'.format(modified_at)
+        with open(filename, 'rb') as f:
+            context.bot.send_document(query.message.chat.id, document=f, caption=caption)
+
+
+updater = Updater(TELEGRAM_TOKEN, use_context=True)
+updater.dispatcher.add_handler(CommandHandler('start', start_handler))
+updater.dispatcher.add_handler(CallbackQueryHandler(callback_handler))
+
+updater.start_polling()
+updater.idle()

parsing/4/main.py

@@ -0,0 +1,111 @@
+import json
+import xlsxwriter
+
+import requests
+from bs4 import BeautifulSoup
+
+
+PAGES_COUNT = 10
+OUT_FILENAME = 'out.json'
+OUT_XLSX_FILENAME = 'out.xlsx'
+
+
+def dump_to_json(filename, data, **kwargs):
+    kwargs.setdefault('ensure_ascii', False)
+    kwargs.setdefault('indent', 1)
+
+    with open(filename, 'w') as f:
+        json.dump(data, f, **kwargs)
+
+
+def dump_to_xlsx(filename, data):
+    if not len(data):
+        return None
+
+    with xlsxwriter.Workbook(filename) as workbook:
+        ws = workbook.add_worksheet()
+        bold = workbook.add_format({'bold': True})
+
+        headers = ['Название товара', 'Цена', 'Ссылка']
+        headers.extend(data[0]['techs'].keys())
+
+        for col, h in enumerate(headers):
+            ws.write_string(0, col, h, cell_format=bold)
+
+        for row, item in enumerate(data, start=1):
+            ws.write_string(row, 0, item['name'])
+            ws.write_string(row, 1, item['amount'])
+            ws.write_string(row, 2, item['url'])
+            for prop_name, prop_value in item['techs'].items():
+                col = headers.index(prop_name)
+                ws.write_string(row, col, prop_value)
+
+
+def get_soup(url, **kwargs):
+    response = requests.get(url, **kwargs)
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.text, features='html.parser')
+    else:
+        soup = None
+    return soup
+
+
+def crawl_products(pages_count):
+    urls = []
+    fmt = 'https://parsemachine.com/sandbox/catalog/?page={page}'
+
+    for page_n in range(1, 1 + pages_count):
+        print('page: {}'.format(page_n))
+
+        page_url = fmt.format(page=page_n)
+        soup = get_soup(page_url)
+        if soup is None:
+            break
+
+        for tag in soup.select('.product-card .title'):
+            href = tag.attrs['href']
+            url = 'https://parsemachine.com{}'.format(href)
+            urls.append(url)
+
+    return urls
+
+
+def parse_products(urls):
+    data = []
+
+    for url in urls:
+        print('\tproduct: {}'.format(url))
+
+        soup = get_soup(url)
+        if soup is None:
+            break
+
+        name = soup.select_one('#product_name').text.strip()
+        amount = soup.select_one('#product_amount').text.strip()
+        techs = {}
+        for row in soup.select('#characteristics tbody tr'):
+            cols = row.select('td')
+            cols = [c.text.strip() for c in cols]
+            techs[cols[0]] = cols[1]
+
+        item = {
+            'name': name,
+            'amount': amount,
+            'techs': techs,
+            'url': url,
+        }
+        data.append(item)
+
+    return data
+
+
+def main():
+    urls = crawl_products(PAGES_COUNT)
+    data = parse_products(urls)
+    dump_to_json(OUT_FILENAME, data)
+    dump_to_xlsx(OUT_XLSX_FILENAME, data)
+
+
+if __name__ == '__main__':
+    main()
+