-
Notifications
You must be signed in to change notification settings - Fork 4
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Anibel/list parser #102
Anibel/list parser #102
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
MANGA_CARD_TAG = '//div[@class = "anime-card grid-6 top-05"]' | ||
TITLE_TAG = "//h1//a/text()" | ||
SOURCE_URL_TAG = "//h1//a/@href" | ||
GENRES_TAG = '//div[@class = "tags"]//a/text()' | ||
THUMBNAIL_IMG_URL_TAG = '//div[contains(@class, "grid-4 left")]//img/@data-src' | ||
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
import logging | ||
|
||
import requests | ||
import scrapy | ||
from lxml import etree | ||
from scrapy.http import HtmlResponse | ||
from twisted.python.failure import Failure | ||
|
||
from apps.core.commands import ParseCommandLogger | ||
|
||
from .consts import GENRES_TAG, MANGA_CARD_TAG, SOURCE_URL_TAG, THUMBNAIL_IMG_URL_TAG, TITLE_TAG | ||
|
||
logging.getLogger(__name__) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Это же ничего не делает |
||
ANIBEL_URL = "https://anibel.net" | ||
|
||
|
||
class AnibelMangaSpider(scrapy.Spider): | ||
management_logger: "ParseCommandLogger" | ||
name = "anibel_manga" | ||
|
||
def __init__(self, *args, logger, **kwargs): | ||
super().__init__(*args, **kwargs) | ||
self.__dict__.update({"management_logger": logger}) | ||
|
||
@property | ||
def logger(self): | ||
return self.management_logger | ||
|
||
def start_requests(self): | ||
self.logger.info("Starting requests") | ||
self.logger.info("=================") | ||
mangas_list = requests.get(f"{ANIBEL_URL}/manga") | ||
if not mangas_list.status_code == 200: | ||
self.logger.error(f"Failed request with code {mangas_list.status_code}") | ||
return | ||
mangas_list = mangas_list.text | ||
|
||
xpath_selector = '//li[@class = "page-item-num"]//a[@class = "page-link"]/text()' | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Это тоже можно вынести в consts.py |
||
html_parser = etree.HTML(mangas_list) | ||
max_page = html_parser.xpath(xpath_selector)[-1] | ||
base_url = f"{ANIBEL_URL}/manga?page=" | ||
pages = [page for page in range(1, int(max_page) + 1)] | ||
urls = [base_url + str(page) for page in pages] | ||
for url in urls: | ||
yield scrapy.Request(url=url, callback=self.parse) | ||
|
||
def request_fallback(self, failure: Failure): | ||
self.logger.error( | ||
f'Request for url "{failure.value.response.url}" ' | ||
f"failed with status {failure.value.response.status}" | ||
) | ||
|
||
def parse(self, response): | ||
mangas = [] | ||
descriptions = response.xpath(MANGA_CARD_TAG).extract() | ||
for description in descriptions: | ||
response = HtmlResponse(url="", body=description, encoding="utf-8") | ||
|
||
full_title = response.xpath(TITLE_TAG).extract_first("") | ||
alt_title = full_title.split(" / ")[1] | ||
alt_title = alt_title.split("[")[0] | ||
title = full_title.split(" / ")[0] | ||
source_url = ANIBEL_URL + response.xpath(SOURCE_URL_TAG).extract_first("") | ||
genres = response.xpath(GENRES_TAG).extract() | ||
image = ANIBEL_URL + response.xpath(THUMBNAIL_IMG_URL_TAG).extract_first("") | ||
mangas.append( | ||
{ | ||
"title": title, | ||
"alt_title": alt_title, | ||
"thumbnail": image, | ||
"genres": genres, | ||
"image": image, | ||
"source_url": source_url, | ||
} | ||
) | ||
self.logger.info('Parsed ,anga "{}"'.format(title)) | ||
|
||
self.logger.info("Processing items...") | ||
self.logger.info("===================") | ||
return mangas |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
import logging | ||
from copy import deepcopy | ||
from typing import List, Tuple | ||
|
||
from django.db import transaction | ||
|
||
from apps.parse.anibel.list_parser.manga_spider import AnibelMangaSpider | ||
from apps.parse.models import Genre, Manga | ||
|
||
logger = logging.getLogger() | ||
|
||
|
||
@transaction.atomic | ||
def bulk_get_or_create(cls, names: List[str]) -> Tuple: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. По-моему эта функция уже есть, глянь в соседних файлах, может можно реюзнуть |
||
objects = [] | ||
for name in names: | ||
objects.append(cls.objects.get_or_create(name=name)) | ||
return tuple(obj for obj, _ in objects) | ||
|
||
|
||
class AnibelPipeline: | ||
@staticmethod | ||
def process_item(item, spider: "AnibelMangaSpider"): | ||
data = deepcopy(item) | ||
|
||
title = data.pop("title") | ||
genres = data.pop("genres") | ||
source_url = data.pop("source_url") | ||
|
||
if not title: | ||
message = f"Error processing {data}: No title name was set" | ||
spider.logger.error(message) | ||
raise KeyError(message) | ||
|
||
genres = bulk_get_or_create(Genre, genres) | ||
|
||
manga, _ = Manga.objects.get_or_create(source_url=source_url) | ||
manga_already = Manga.objects.filter(source_url=source_url) | ||
if manga_already.exists(): | ||
manga_already.update(title=title, **data) | ||
manga = manga_already.first() | ||
spider.logger.info(f'Updated item "{manga}"') | ||
else: | ||
manga = Manga.objects.create(title=title, source_url=source_url, **data) | ||
spider.logger.info(f'Created item "{manga}"') | ||
|
||
manga.genres.add(*genres) | ||
|
||
return item |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
from apps.parse.anibel.list_parser.pipelines import AnibelPipeline | ||
|
||
BOT_NAME = "anibel" | ||
|
||
SPIDER_MODULES = ["apps.parse.anibel.list_parser"] | ||
NEWSPIDER_MODULE = "apps.parse.anibel.list_parser" | ||
|
||
ROBOTSTXT_OBEY = True | ||
|
||
DOWNLOAD_DELAY = 2 | ||
|
||
ITEM_PIPELINES = { | ||
AnibelPipeline: 300, | ||
dhvcc marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
AUTOTHROTTLE_ENABLED = True | ||
AUTOTHROTTLE_START_DELAY = 2 | ||
AUTOTHROTTLE_MAX_DELAY = 15 | ||
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Что-то мне кажется, что разметка сайта поменялась, потому что эти XPATH не работают.
У них есть сейчас сайт old.anibel.net и там работает