-
Notifications
You must be signed in to change notification settings - Fork 274
/
grab_response_body_processing.py
92 lines (80 loc) · 3.36 KB
/
grab_response_body_processing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# coding: utf-8
from grab import GrabMisuseError
from test.util import temp_dir, build_grab
from test.util import BaseGrabTestCase
import os
class GrabSimpleTestCase(BaseGrabTestCase):
def setUp(self):
self.server.reset()
def test_body_inmemory_false(self):
with temp_dir() as tmp_dir:
g = build_grab()
g.setup(body_inmemory=False)
self.assertRaises(GrabMisuseError, lambda: g.go(self.server.get_url()))
self.server.response['get.data'] = b'foo'
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=tmp_dir)
g.go(self.server.get_url())
self.assertTrue(os.path.exists(g.response.body_path))
self.assertTrue(tmp_dir in g.response.body_path)
self.assertEqual(b'foo', open(g.response.body_path, 'rb').read())
# pylint: disable=protected-access
self.assertEqual(g.response._bytes_body, None)
# pylint: enable=protected-access
old_path = g.response.body_path
g.go(self.server.get_url())
self.assertTrue(old_path != g.response.body_path)
with temp_dir() as tmp_dir:
self.server.response['get.data'] = 'foo'
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=tmp_dir)
g.setup(body_storage_filename='music.mp3')
g.go(self.server.get_url())
self.assertTrue(os.path.exists(g.response.body_path))
self.assertTrue(tmp_dir in g.response.body_path)
self.assertEqual(b'foo', open(g.response.body_path, 'rb').read())
self.assertEqual(os.path.join(tmp_dir, 'music.mp3'),
g.response.body_path)
self.assertEqual(g.response.body, b'foo')
# pylint: disable=protected-access
self.assertEqual(g.response._bytes_body, None)
# pylint: enable=protected-access
def test_body_inmemory_true(self):
g = build_grab()
self.server.response['data'] = b'bar'
g.go(self.server.get_url())
# pylint: disable=protected-access
self.assertEqual(g.response._bytes_body, b'bar')
# pylint: enable=protected-access
def test_assign_unicode_to_body(self):
g = build_grab()
g.doc.body = b'abc'
g.doc.body = b'def'
with self.assertRaises(GrabMisuseError):
g.doc.body = u'Спутник'
def test_empty_response(self):
self.server.response['data'] = b''
g = build_grab()
g.go(self.server.get_url())
# pylint: disable=pointless-statement
g.doc.tree # should not raise exception
# pylint: enable=pointless-statement
#def test_emoji_processing(self):
# #html = u'''
# #<html><body>
# # <span class="a-color-base"> 👍🏻 </span>
# #</body></html>
# #'''.encode('utf-8')
# g = build_grab()
# #print('>>',g.doc('//span').text(),'<<')
# #import grab
# #g = grab.Grab()
# #g.go('https://github.com/lorien/grab/issues/199#issuecomment-269854859')
# g.go('https://en.wikipedia.org/wiki/Emoji')
# g.doc.select("//*")
def test_doc_tree_notags_document(self):
data = b'test'
g = build_grab(data)
self.assertEqual(g.doc('//html').text(), 'test')