Skip to content

Commit

Permalink
Merge pull request #2084 from dipu-bd/dev
Browse files Browse the repository at this point in the history
Version 3.3.0
  • Loading branch information
dipu-bd committed Sep 8, 2023
2 parents 2dae4e9 + da1d10d commit b992197
Show file tree
Hide file tree
Showing 7 changed files with 373 additions and 366 deletions.
4 changes: 3 additions & 1 deletion .github/contribs.json
Expand Up @@ -78,5 +78,7 @@
"ismaelcompsci": null,
"ismaelcompsci@gmail.com": null,
"needKVAS": null,
"43537033+needKVAS@users.noreply.github.com": null
"43537033+needKVAS@users.noreply.github.com": null,
"Aleksa Savic": null,
"savicaleksa.kg@gmail.com": null
}
715 changes: 360 additions & 355 deletions README.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion lncrawl/VERSION
@@ -1 +1 @@
3.2.10
3.3.0
2 changes: 1 addition & 1 deletion lncrawl/binders/calibre.py
Expand Up @@ -59,7 +59,7 @@ def epub_to_calibre(app, epub_file, out_fmt):
"--comments",
app.crawler.novel_synopsis,
"--language",
app.crawler.novel_language,
app.crawler.language,
"--tags",
",".join(app.crawler.novel_tags),
"--series",
Expand Down
2 changes: 1 addition & 1 deletion sources/_index.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion sources/en/1/1stkissnovel.py
Expand Up @@ -50,7 +50,7 @@ def read_novel_info(self):

img_src = soup.select_one(".summary_image a img")
if img_src:
self.novel_cover = self.absolute_url(img_src["data-lazy-src"])
self.novel_cover = self.absolute_url(img_src["src"])
logger.info("Novel cover: %s", self.novel_cover)

self.novel_author = " ".join(
Expand Down
12 changes: 6 additions & 6 deletions sources/zh/69shu.py → sources/zh/69shuba.py
Expand Up @@ -12,9 +12,9 @@
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Content-Type": "application/x-www-form-urlencoded",
"Origin": "https://www.69shu.com",
"Origin": "https://www.69shuba.com", # Updated to the new domain
"DNT": "1",
"Alt-Used": "www.69shu.com",
"Alt-Used": "www.69shuba.com", # Updated to the new domain
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
Expand All @@ -24,11 +24,11 @@
}

logger = logging.getLogger(__name__)
search_url = "https://www.69shu.com/modules/article/search.php"
search_url = "https://www.69shuba.com/modules/article/search.php" # Updated to the new domain


class sixnineshu(Crawler):
base_url = "https://www.69shu.com/"
base_url = ["https://www.69shuba.com/", "https://www.69shu.com/"] # Updated to the new domain

def get_soup(self, url):
"""overwrite the get_soup function to set the encoding"""
Expand All @@ -40,7 +40,7 @@ def get_soup(self, url):
def search_novel(self, query):
query = urllib.parse.quote(query.encode("gbk"))
data = requests.post(
"https://www.69shu.com/modules/article/search.php",
"https://www.69shuba.com/modules/article/search.php", # Updated to the new domain
headers=headers,
data=f"searchkey={query}&searchtype=all",
)
Expand Down Expand Up @@ -105,7 +105,7 @@ def read_novel_info(self):
self.novel_tags = [possible_tag.text.strip()]
logger.info("Novel Tag: %s", self.novel_tags)

# https://www.69shu.com/txt/A43616.htm -> https://www.69shu.com/A43616/
# https://www.69shuba.com/txt/A43616.htm -> https://www.69shuba.com/A43616/
soup = self.get_soup(self.novel_url.replace("/txt/", "/").replace(".htm", "/"))

for li in soup.select("div.catalog ul li"):
Expand Down

0 comments on commit b992197

Please sign in to comment.