Merge branch 'feature/import-export' into develop

w3af/core/data/dc/headers.py

@@ -51,8 +51,8 @@ def from_string(cls, headers_str):
             Content-Length: 123
         """
         res = []
-        splitted_str = headers_str.split('\r\n')
-        for one_header_line in splitted_str:
+        split_str = headers_str.split('\r\n')
+        for one_header_line in split_str:
 
             if not one_header_line:
                 continue
@@ -170,12 +170,16 @@ def __str__(self):
         :return: string representation of the Headers() object.
         """
         header_str_unicode = self._to_str_with_separators(u': ', u'\r\n')
-        header_str_unicode += u'\r\n'
-
+        if header_str_unicode:
+            header_str_unicode += u'\r\n'
+
         return header_str_unicode.encode('utf-8')
 
     def __unicode__(self):
         """
         :see: __str__ documentation.
         """
-        return self._to_str_with_separators(u': ', u'\r\n') + u'\r\n'
+        headers_unicode = self._to_str_with_separators(u': ', u'\r\n')
+        if headers_unicode:
+            headers_unicode += u'\r\n'
+        return headers_unicode

w3af/core/data/dc/multipart_container.py

@@ -84,7 +84,8 @@ def from_postdata(cls, headers, post_data):
                 else:
                     attrs = {'type': INPUT_TYPE_FILE,
                              'name': key.name,
-                             'value': key.file.read()}
+                             'value': key.file.read(),
+                             'filename': key.filename}
                     form_params.add_field_by_attrs(attrs)
                     form_params.set_file_name(key.name, key.filename)
 

w3af/core/data/dc/tests/test_multipart.py

@@ -92,8 +92,7 @@ def test_multipart_test_from_string(self):
         self.assertIn('file', mpc)
 
         self.assertEqual(mpc['MAX_FILE_SIZE'], ['2097152'])
-        # We don't store the file content
-        self.assertEqual(mpc['file'], [''])
+        self.assertTrue(mpc['file'][0].startswith('GIF89'))
 
         self.assertEqual(mpc.get_file_vars(), ['file'])
         self.assertEqual(mpc.get_parameter_type('MAX_FILE_SIZE'), 'text')
@@ -163,8 +162,7 @@ def test_multipart_3570(self):
         self.assertIn('userfile', mpc)
 
         self.assertEqual(mpc['MAX_FILE_SIZE'], ['2097152'])
-        # We don't store the file content
-        self.assertEqual(mpc['userfile'], [''])
+        self.assertTrue(mpc['userfile'][0].startswith('GIF89'))
 
         self.assertEqual(mpc.get_file_vars(), ['userfile'])
         self.assertEqual(mpc.get_parameter_type('MAX_FILE_SIZE'), 'text')

w3af/core/data/parsers/doc/http_request_parser.py

@@ -35,20 +35,20 @@ def check_version_syntax(version):
     :return: True if the syntax of the version section of HTTP is valid; else
              raise an exception.
     """
-    splitted_version = version.split('/')
+    split_version = version.split('/')
 
-    if len(splitted_version) != 2:
+    if len(split_version) != 2:
         msg = 'The HTTP request has an invalid version token: "%s"'
         raise BaseFrameworkException(msg % version)
 
-    elif len(splitted_version) == 2:
+    elif len(split_version) == 2:
 
-        if splitted_version[0].lower() != 'http':
-            msg = 'The HTTP request has an invalid HTTP token in the version'\
-                  ' specification: "%s"'
+        if split_version[0].lower() != 'http':
+            msg = ('The HTTP request has an invalid HTTP token in the version'
+                   ' specification: "%s"')
             raise BaseFrameworkException(msg % version)
 
-        if splitted_version[1] not in SUPPORTED_VERSIONS:
+        if split_version[1] not in SUPPORTED_VERSIONS:
             fmt = 'HTTP request version "%s" is unsupported'
             raise BaseFrameworkException(fmt % version)
 
@@ -80,6 +80,16 @@ def check_uri_syntax(uri, host=None):
     return res
 
 
+def raw_http_request_parser(raw_http_request):
+    """
+    :param raw_http_request: An HTTP request with headers and body as a string
+    :return: A FuzzableRequest object with all the corresponding information
+             that was sent in head and postdata
+    """
+    head, postdata = raw_http_request.split('\r\n\r\n', 1)
+    return http_request_parser(head, postdata)
+
+
 def http_request_parser(head, postdata):
     """
     This function parses HTTP Requests from a string to a FuzzableRequest.

w3af/core/data/parsers/doc/tests/test_html.py

@@ -151,7 +151,7 @@ def test_inputs_in_out_form(self):
         self.assertEquals([''], f['foo6'])            # checkbox input
         self.assertEquals(['bar'], f['foo7'])         # hidden input
         self.assertEquals([''], f['foo4'])            # submit input
-        self.assertEquals([''], f['foo3'])            # file input
+        self.assertEquals(['bar'], f['foo3'])         # file input
 
         # 2nd body
         body2 = HTML_DOC % \

w3af/core/data/parsers/utils/form_params.py

@@ -295,7 +295,10 @@ def form_field_factory(self, attributes):
                 form_field = CheckboxFormField(input_name, [input_value])
 
         elif input_type == INPUT_TYPE_FILE:
-            form_field = FileFormField(input_name)
+            file_name = get_value_by_key(attributes, 'filename')
+            form_field = FileFormField(input_name,
+                                       value=input_value,
+                                       file_name=file_name)
 
         else:
             form_field = GenericFormField(input_type, input_name, input_value,

w3af/core/data/request/fuzzable_request.py

@@ -19,9 +19,8 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 """
-import csv
 import string
-import cStringIO
+import base64
 
 from urllib import unquote
 from itertools import chain
@@ -157,7 +156,7 @@ def from_http_response(cls, http_response):
         return cls(http_response.get_uri(), method='GET', cookie=cookie)
 
     @classmethod
-    def from_urllib2_request(cls, request):
+    def from_http_request(cls, request):
         """
         :param request: The instance we'll use as base
         :return: An instance of FuzzableRequest based on a urllib2 HTTP request
@@ -190,35 +189,22 @@ def from_form(cls, form, headers=None):
 
         return r
 
-    def to_csv(self):
+    def to_base64(self):
         """
-        Generic version of how fuzzable requests are exported:
-            METHOD,URL,POST_DATA
-
-        Example:
-            GET,http://localhost/index.php?abc=123&def=789,
-            POST,http://localhost/index.php,abc=123&def=789
-
-        :return: a csv str representation of the request
+        :return: The whole HTTP request serialized and encoded as base64
         """
-        #
-        # TODO: Why don't we export headers and cookies?
-        #
-        output = cStringIO.StringIO()
-        writer = csv.writer(output, quoting=csv.QUOTE_ALL)
-        writer.writerow((self.get_method(), self.get_uri(), self._post_data))
-        output.seek(0)
-        return str(output.read())[:-2]
+        raw_http_request = self.dump()
+        return base64.b64encode(raw_http_request)
 
     @classmethod
-    def from_csv(cls, csv_line):
+    def from_base64(cls, base64_data):
         """
-        :param csv_line: A string representing a line in a CSV file
+        :param base64_data: A string generated by to_base64
         :return: A FuzzableRequest instance
         """
-        row = csv.reader([csv_line], quoting=csv.QUOTE_ALL).next()
-        return FuzzableRequest.from_parts(row[1], method=row[0],
-                                          post_data=row[2])
+        from w3af.core.data.parsers.doc.http_request_parser import raw_http_request_parser
+        raw_http_request = base64.b64decode(base64_data)
+        return raw_http_request_parser(raw_http_request)
 
     def sent(self, smth_instng):
         """
@@ -494,13 +480,24 @@ def get_all_headers(self):
         for k, v in chain(self._headers.items(),
                           self.get_post_data_headers().items()):
 
-            # Ignore any keys which are already defined in the user-specified
-            # headers
-            kvalue, kreal = wire_headers.iget(k, None)
-            if kvalue is not None:
-                continue
-
-            wire_headers[k] = v
+            # Please note that here we're overwriting the headers from the
+            # fuzzable request with the headers from the data container,
+            # the overwriting is done in this order due to the order in the
+            # chain() items above
+            #
+            # I found a bug where I loaded a request from spider_man, saved
+            # it using dump() and then tried to load it again and failed because
+            # of this overwriting not being done (the multipart boundary was
+            # incorrect).
+            #
+            # Keep that in mind in case you want to change this overwriting!
+            #
+            # Overwrite the existing one, case insensitive style
+            _, stored_header_name = wire_headers.iget(k, None)
+            if stored_header_name is not None:
+                wire_headers[stored_header_name] = v
+            else:
+                wire_headers[k] = v
 
         return wire_headers
 

w3af/core/data/request/request_mixin.py

@@ -33,7 +33,9 @@ class RequestMixIn(object):
 
     def dump(self, ignore_headers=()):
         """
-        :return: The HTTP request as it would be sent to the wire.
+        :return: The HTTP request as it would be sent to the wire, with a minor
+                 change, instead of using the path in the second token of the
+                 request we use the URL, this is just a user-friendly feature
 
                  Please note that we're returning a byte-string, with the
                  special characters in the headers and URL encoded as expected

w3af/core/data/request/tests/test_fuzzable_request.py

@@ -86,7 +86,7 @@ def test_dump_case03(self):
         self.assertEqual(fr.dump(), expected)
 
     def test_dump_mangle(self):
-        fr = FuzzableRequest(URL("http://www.w3af.com/"),\
+        fr = FuzzableRequest(URL("http://www.w3af.com/"),
                              headers=Headers([('Host', 'www.w3af.com')]))
 
         expected = u'\r\n'.join([u'GET http://www.w3af.com/ HTTP/1.1',
@@ -107,24 +107,20 @@ def test_dump_mangle(self):
         self.assertEqual(fr.dump(), expected)
 
     def test_export_import_without_post_data(self):
-        fr = FuzzableRequest(URL("http://www.w3af.com/"))
-        self.assertEqual(fr.to_csv(), '"GET","http://www.w3af.com/",""')
+        fr = FuzzableRequest(URL('http://www.w3af.com/'))
 
-        imported_fr = fr.from_csv(fr.to_csv())
+        imported_fr = FuzzableRequest.from_base64(fr.to_base64())
         self.assertEqual(imported_fr, fr)
 
     def test_export_import_with_post_data(self):
         dc = KeyValueContainer(init_val=[('a', ['1'])])
-        fr = FuzzableRequest(URL("http://www.w3af.com/"), post_data=dc)
-
-        self.assertEqual(fr.to_csv(), '"GET","http://www.w3af.com/","a=1"')
+        fr = FuzzableRequest(URL('http://www.w3af.com/'), post_data=dc)
 
-        raise SkipTest('Failing because we do NOT export headers')
-        imported_fr = fr.from_csv(fr.to_csv())
+        imported_fr = FuzzableRequest.from_base64(fr.to_base64())
         self.assertEqual(imported_fr, fr)
 
     def test_equal(self):
-        u = URL("""http://www.w3af.com/""")
+        u = URL("http://www.w3af.com/")
         fr1 = FuzzableRequest(u)
         fr2 = FuzzableRequest(u)
         self.assertEqual(fr1, fr2)