Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

clean up tests for 0.8

  • Loading branch information...
commit b4d84a52d93cb2b33fe8a834c5011980d2dee366 1 parent b25e57d
@jamesturk jamesturk authored
Showing with 9 additions and 18 deletions.
  1. +2 −3 scrapelib/__init__.py
  2. +7 −15 scrapelib/tests/test_scraper.py
View
5 scrapelib/__init__.py
@@ -287,8 +287,6 @@ def __init__(self,
retry_attempts=0,
retry_wait_seconds=5,
header_func=None,
- cache_obj=None, # deprecated
- cache_write_only=True, # deprecated
):
super(Scraper, self).__init__()
@@ -343,7 +341,8 @@ def request(self, method, url, **kwargs):
timeout = kwargs.pop('timeout', self.timeout)
if self._header_func:
- headers = Headers(self._header_func(url))
+ headers = requests.structures.CaseInsensitiveDict(
+ self._header_func(url))
else:
headers = {}
headers = requests.sessions.merge_kwargs(headers, self.headers)
View
22 scrapelib/tests/test_scraper.py
@@ -38,24 +38,16 @@ def request_200(method, url, *args, **kwargs):
def test_fields():
# timeout=0 means None
- s = Scraper(user_agent='secret-agent',
- requests_per_minute=100,
+ s = Scraper(requests_per_minute=100,
follow_robots=False,
- disable_compression=True,
raise_errors=False,
- timeout=0,
retry_attempts=-1, # will be 0
- retry_wait_seconds=100,
- cache_write_only=False)
- assert_equal(s.user_agent, 'secret-agent')
+ retry_wait_seconds=100)
assert s.requests_per_minute == 100
assert s.follow_robots == False
- assert s.disable_compression
assert s.raise_errors == False
- assert s.timeout is None # 0 becomes None
assert s.retry_attempts == 0 # -1 becomes 0
assert s.retry_wait_seconds == 100
- assert s.cache_write_only == False
def test_get():
@@ -169,10 +161,9 @@ def test_500():
def test_caching():
cache_dir = tempfile.mkdtemp()
- #s = Scraper(requests_per_minute=0, follow_robots=False,
- # cache_obj=FileCache(cache_dir), cache_write_only=False)
- s = Scraper(requests_per_minute=0, follow_robots=False,
- cache_obj=MemoryCache(), cache_write_only=False)
+ s = Scraper(requests_per_minute=0, follow_robots=False)
+ s.cache_storage = MemoryCache()
+ s.cache_write_only = False
resp = s.urlopen(HTTPBIN + 'status/200')
assert not resp.response.fromcache
@@ -306,7 +297,8 @@ def side_effect(*args, **kwargs):
def test_disable_compression():
- s = Scraper(disable_compression=True)
+ s = Scraper()
+ s.disable_compression = True
# compression disabled
data = s.urlopen(HTTPBIN + 'headers')
Please sign in to comment.
Something went wrong with that request. Please try again.