-
-
Notifications
You must be signed in to change notification settings - Fork 3
/
test_gateways.py
31 lines (26 loc) · 1.25 KB
/
test_gateways.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# coding: utf-8
from gummy import journals
from gummy.utils import get_driver
from gummy.journals import whichJournal
from gummy.utils import check_environ
def _test_gateways(db, gateway, journal_type="nature", **gatewaykwargs):
url = db.journals.get(journal_type)
# Crawler with gateway.
gateway_crawler = journals.get(identifier=journal_type, gateway=gateway)
is_ok, _ = check_environ(
required_kwargs=gateway_crawler.gateway.get_required_kwargs(journal_type=journal_type),
required_env_varnames=gateway_crawler.gateway.get_required_env_varnames(journal_type=journal_type),
verbose=1,
**gatewaykwargs,
)
if is_ok:
with get_driver() as driver:
gateway_title, gateway_contents = gateway_crawler.get_contents(url=url, driver=driver, **gatewaykwargs)
# Crawler without gateway.
useless_crawler = journals.get(identifier=journal_type, gateway="useless")
with get_driver() as driver:
useless_title, useless_contents = useless_crawler.get_contents(url=url, driver=driver)
assert useless_title == gateway_title
assert len(useless_contents) < len(gateway_contents)
def test_utokyo_gateways(db):
_test_gateways(db=db, gateway="utokyo")