-
Notifications
You must be signed in to change notification settings - Fork 0
/
novel_meta_crawler.py
65 lines (48 loc) · 1.96 KB
/
novel_meta_crawler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import os
import datetime as dt
import requests
from bs4 import BeautifulSoup
from tqdm.auto import tqdm
from dotenv import load_dotenv
import pymongo
import pprint
load_dotenv(verbose=True)
IP = os.getenv("IP")
PORT = int(os.getenv("PORT"))
DATABASE = os.getenv("DATABASE")
COL = os.getenv("META_COLLECTION")
conn = pymongo.MongoClient("mongodb://{}:{}".format(IP,PORT))
db = conn[DATABASE]
col = db[COL]
def main():
now = dt.datetime.now()
year = now.year
month = now.month
new = [] # 새로운 소설 정보 저장
exception = [] # 저장이 안된 소설들
url = "https://munjang.or.kr/board.es?mid=a20103000000&bid=0003&nPage=1&b_list_cnt=12&ord=&dept_cd=&tag=&list_no=&listNo=&act=list&view_sdate_param=&keyField=&keyWord="
plain = requests.get(url).text
contents = BeautifulSoup(plain,'html.parser')
for c in contents.find_all("li", {"class" : "list_li"}):
uploadDate = list(map(int,c.find("span",{"class":"date"}).find("span").text.split("-")))
if uploadDate[0] == year and uploadDate[1] == month: #업로드 일자가 해당 년, 월과 같은 경우 크롤링
global novels
novels = {}
try:
info = c.find('strong').text.split(" - ")
novels['title'] = info[1]
novels["author"] = info[0]
except:
exception.extend(info)
novels['title'] = info[0]
novels['author'] = "None"
print("info : ", info[0] , "status : FINISHED")
novels["links"] = "https://munjang.or.kr" + c.find("a")["href"]
novels["crawling_date"] = now.strftime("%Y-%m-%d %H:%M:%S")
if col.find_one({'$and' : [{'title':novels['title']},{'author':novels['author']}] }): # 중복제거
pass
else:
col.insert_one(novels)
print("INSERT NOVELS : ", novels.get("title"))
if __name__ == "__main__":
main()