Fix processing of upload data in debug=True mode

Author: lorien

grab/base.py

@@ -186,7 +186,8 @@ def default_config():
 
 class Grab(DeprecatedThings):
 
-    __slots__ = ('request_head', 'request_log', 'request_body',
+    __slots__ = ('request_head', 'request_body',
+                 #'request_log',
                  'proxylist', 'config',
                  'transport',
                  'transport_param', 'request_method', 'request_counter',
@@ -200,7 +201,8 @@ class Grab(DeprecatedThings):
 
     # Attributes which should be processed when clone
     # of Grab instance is creating
-    clonable_attributes = ('request_head', 'request_log', 'request_body',
+    clonable_attributes = ('request_head', 'request_body',
+                           #'request_log',
                            'proxylist')
 
     # Complex config items which points to mutable objects
@@ -272,7 +274,7 @@ def reset(self):
         """
 
         self.request_head = None
-        self.request_log = None
+        #self.request_log = None
         self.request_body = None
         self.request_method = None
         self.transport.reset()
@@ -580,7 +582,7 @@ def copy_request_data(self):
         # TODO: Maybe request object?
         self.request_head = self.transport.request_head
         self.request_body = self.transport.request_body
-        self.request_log = self.transport.request_log
+        #self.request_log = self.transport.request_log
 
     def setup_document(self, content, **kwargs):
         """
@@ -666,15 +668,15 @@ def save_dumps(self):
             thread_name = '-%s' % thread_name
         file_name = os.path.join(self.config['log_dir'], '%02d%s.log' % (
             self.request_counter, thread_name))
-        with open(file_name, 'w') as out:
-            out.write('Request headers:\n')
+        with open(file_name, 'wb') as out:
+            out.write(b'Request headers:\n')
             out.write(self.request_head)
-            out.write('\n')
-            out.write('Request body:\n')
+            out.write(b'\n')
+            out.write(b'Request body:\n')
             out.write(self.request_body)
-            out.write('\n\n')
-            out.write('Response headers:\n')
-            out.write(self.doc.head.decode('ascii'))
+            out.write(b'\n\n')
+            out.write(b'Response headers:\n')
+            out.write(self.doc.head)
 
         file_extension = 'html'
         file_name = os.path.join(self.config['log_dir'], '%02d%s.%s' % (

grab/transport/curl.py

@@ -104,9 +104,9 @@ def reset(self):
         self.verbose_logging = False
 
         # Maybe move to super-class???
-        self.request_head = ''
-        self.request_body = ''
-        self.request_log = ''
+        self.request_head = b''
+        self.request_body = b''
+        #self.request_log = ''
 
     def header_processor(self, chunk):
         """
@@ -156,9 +156,18 @@ def debug_processor(self, _type, text):
         """
 
         if _type == pycurl.INFOTYPE_HEADER_OUT:
+            if isinstance(text, six.text_type):
+                text = text.encode('utf-8')
             self.request_head += text
 
         if _type == pycurl.INFOTYPE_DATA_OUT:
+            # Untill 7.19.5.2 version
+            # pycurl gives unicode in `text` variable
+            # WTF??? Probably that codes would fails
+            # or does unexpected things if you use
+            # pycurl<7.19.5.2
+            if isinstance(text, six.text_type):
+                text = text.encode('utf-8')
             self.request_body += text
 
         """

test/grab_debug.py

@@ -87,9 +87,9 @@ def test_log_dir_request_content_headers_and_post(self):
             g.go(self.server.get_url())
             self.assertEqual(sorted(os.listdir(tmp_dir)), ['01.html', '01.log'])
             log_file_content = open(os.path.join(tmp_dir, '01.log')).read()
-            if not 'x-name' in log_file_content.lower():
-                print('CONTENT OF 01.log:')
-                print(log_file_content)
+            #if not 'x-name' in log_file_content.lower():
+            #    print('CONTENT OF 01.log:')
+            #    print(log_file_content)
             self.assertTrue('x-name' in log_file_content.lower())
             self.assertTrue('xxx=post' in log_file_content.lower())
 
@@ -100,6 +100,13 @@ def test_debug_post(self):
         g.go(self.server.get_url())
         self.assertEqual(b'x', g.doc.body)
 
+    def test_debug_nonascii_post(self):
+        g = build_grab(debug=True)
+        g.setup(post=u'фыва'.encode('cp1251'))
+        g.go(self.server.get_url())
+        g.setup(multipart_post=[('x', u'фыва'.encode('cp1251'))])
+        g.go(self.server.get_url())
+
     def test_debug_post_integer_bug(self):
         g = build_grab(debug_post=True)
         g.setup(post={'foo': 3})