Merge branch 'develop'

extras/docker/Dockerfile

@@ -50,7 +50,7 @@ RUN pip install --upgrade pip
 #
 #tblib==0.2.0
 #
-RUN pip install clamd==1.0.1 PyGithub==1.21.0 GitPython==0.3.2.RC1 pybloomfiltermmap==0.3.14 \
+RUN pip install clamd==1.0.1 PyGithub==1.21.0 GitPython==2.1.3 pybloomfiltermmap==0.3.14 \
         esmre==0.3.1 phply==0.9.1 nltk==3.0.1 chardet==2.1.1 pdfminer==20140328 \
         futures==2.1.5 pyOpenSSL==0.15.1 scapy-real==2.2.0-dev guess-language==0.2 cluster==1.1.1b3 \
         msgpack-python==0.4.4 python-ntlm==1.0.1 halberd==0.2.4 darts.util.lru==0.5 \

w3af/core/controllers/delay_detection/exact_delay_controller.py

@@ -47,7 +47,7 @@ class ExactDelayController(DelayMixIn):
     # you'll only get slower scans when there is a vulnerability, which is not
     # the most common case
     #
-    DELAY_SECONDS = [12, 8, 15, 20]
+    DELAY_SECONDS = [12, 8, 15, 20, 4, 4, 4]
 
     def __init__(self, mutant, delay_obj, uri_opener):
         """

w3af/core/controllers/dependency_check/requirements.py

@@ -27,7 +27,7 @@
 
 CORE_PIP_PACKAGES = [PIPDependency('pyclamd', 'pyClamd', '0.3.15'),
                      PIPDependency('github', 'PyGithub', '1.21.0'),
-                     PIPDependency('git.util', 'GitPython', '0.3.2.RC1'),
+                     PIPDependency('git.util', 'GitPython', '2.1.3'),
                      PIPDependency('pybloomfilter', 'pybloomfiltermmap', '0.3.14'),
                      PIPDependency('esmre', 'esmre', '0.3.1'),
                      PIPDependency('phply', 'phply', '0.9.1'),

w3af/core/controllers/misc/contains_source_code.py

@@ -20,6 +20,7 @@
 
 """
 import re
+import string
 
 from w3af.core.data.esmre.multi_re import multi_re
 
@@ -133,4 +134,16 @@ def is_false_positive(http_response, match, detected_langs):
             if 'javascript' in http_response.content_type:
                 return True
 
+    # Avoid some false positives in large binary files where we might
+    # have <% , then 182837 binary chars, and finally %>.
+    printable = 0.0
+    ratio = 0.9
+
+    for char in match_str:
+        if char in string.printable:
+            printable += 1
+
+    if (printable / len(match_str)) < ratio:
+        return True
+
     return False

w3af/core/controllers/misc/tests/test_contains_source_code.py

@@ -20,16 +20,20 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 """
+import os
 import unittest
 
 from w3af.core.controllers.misc.contains_source_code import contains_source_code
 from w3af.core.controllers.misc.contains_source_code import PHP, PYTHON, RUBY, JAVA
 from w3af.core.data.url.HTTPResponse import HTTPResponse
 from w3af.core.data.parsers.doc.url import URL
 from w3af.core.data.dc.headers import Headers
+from w3af import ROOT_PATH
 
 
 class TestContainsSourceCode(unittest.TestCase):
+    TEST_FILE = os.path.join(ROOT_PATH, 'core', 'controllers', 'misc', 'tests',
+                             'data', 'code-detect-false-positive.jpg')
 
     def create_response(self, body, content_type=None):
         content_type = content_type if content_type is not None else 'text/html'
@@ -174,3 +178,10 @@ def test_code_false_positive_java_02(self):
                  ''')
         match, lang = contains_source_code(source)
         self.assertEqual(match, None)
+
+    def test_code_false_positive_image(self):
+        no_source = self.create_response(file(self.TEST_FILE).read(),
+                                         content_type='image/jpeg')
+        match, lang = contains_source_code(no_source)
+        self.assertEqual(match, None)
+

w3af/core/controllers/sql_tools/blind_sqli_response_diff.py

@@ -40,6 +40,12 @@ class BlindSqliResponseDiff(object):
 
     SPECIAL_CHARS = '"\'=()'
     SYNTAX_ERROR = u"a'b\"c'd\""
+    CONFIRMATION_ROUNDS = 3
+
+    NUMERIC = 'numeric'
+    STRING_SINGLE = 'string_single'
+    STRING_DOUBLE = 'string_double'
+    STATEMENT_TYPES = [NUMERIC, STRING_SINGLE, STRING_DOUBLE]
 
     def __init__(self, uri_opener):
         # User configured variables
@@ -64,16 +70,40 @@ def is_injectable(self, mutant):
 
         :return: A vulnerability object or None if nothing is found
         """
-        statements = self._get_statements(mutant)
-        for statement_type in statements:
-            try:
-                vuln = self._find_bsql(mutant,
-                                       statements[statement_type],
-                                       statement_type)
-            except HTTPRequestException:
-                continue
-            else:
-                if vuln:
+        for statement_type in self.STATEMENT_TYPES:
+
+            #
+            # These confirmation rounds are just to reduce false positives
+            # Note that this has a really low impact on the number of HTTP
+            # requests sent during a scan because it is only run if there
+            # seems to be a blind SQL injection
+            #
+            confirmations = 0
+            for _ in xrange(self.CONFIRMATION_ROUNDS):
+
+                # Get fresh statements with new random numbers for each
+                # confirmation round.
+                statements = self._get_statements(mutant)
+
+                try:
+                    vuln = self._find_bsql(mutant,
+                                           statements[statement_type],
+                                           statement_type)
+                except HTTPRequestException:
+                    continue
+
+                # One of the confirmation rounds yields "no vuln found"
+                if vuln is None:
+                    msg = 'Confirmation round %s for %s failed.' % (confirmations, statement_type)
+                    self.debug(msg, mutant)
+                    break
+
+                # One confirmation round succeeded
+                msg = 'Confirmation round %s for %s succeeded.' % (confirmations, statement_type)
+                self.debug(msg, mutant)
+                confirmations += 1
+
+                if confirmations == self.CONFIRMATION_ROUNDS:
                     return vuln
 
         return None
@@ -91,17 +121,17 @@ def _get_statements(self, mutant, exclude_numbers=[]):
         # Numeric/Datetime
         true_stm = '%(num)s OR %(num)s=%(num)s OR %(num)s=%(num)s ' % num_dict
         false_stm = '%i AND %i=%i ' % (rnd_num, rnd_num, rnd_num_plus_one)
-        res['numeric'] = (true_stm, false_stm)
+        res[self.NUMERIC] = (true_stm, false_stm)
 
         # Single quotes
         true_stm = "%(num)s' OR '%(num)s'='%(num)s' OR '%(num)s'='%(num)s" % num_dict
         false_stm = "%i' AND '%i'='%i" % (rnd_num, rnd_num, rnd_num_plus_one)
-        res['string_single'] = (true_stm, false_stm)
+        res[self.STRING_SINGLE] = (true_stm, false_stm)
 
         # Double quotes
         true_stm = '%(num)s" OR "%(num)s"="%(num)s" OR "%(num)s"="%(num)s' % num_dict
         false_stm = '%i" AND "%i"="%i' % (rnd_num, rnd_num, rnd_num_plus_one)
-        res['string_double'] = (true_stm, false_stm)
+        res[self.STRING_DOUBLE] = (true_stm, false_stm)
 
         return res
 
@@ -119,8 +149,9 @@ def _remove_all_special_chars(self, sql_statement):
 
         # escape double white spaces, not sure if this has any logical value
         # in the search engine, but just in case...
-        while '  ' in sql_statement:
-            sql_statement = sql_statement.replace('  ', ' ')
+        double_spaces = '  '
+        while double_spaces in sql_statement:
+            sql_statement = sql_statement.replace(double_spaces, ' ')
 
         return sql_statement
 
@@ -142,17 +173,16 @@ def _find_bsql(self, mutant, statement_tuple, statement_type):
         _, body_false_response = send_clean(mutant)
 
         if body_true_response == body_false_response:
-            #
-            #    There is NO CHANGE between the true and false responses.
-            #    NO WAY I'm going to detect a blind SQL injection using
-            #    response diffs in this case.
-            #
+            msg = ('There is NO CHANGE between the true and false responses.'
+                   ' NO WAY w3af is going to detect a blind SQL injection'
+                   ' using response diffs in this case.')
+            self.debug(msg, mutant)
             return None
 
         compare_diff = False
 
         self.debug('[%s] Comparing body_true_response and'
-                   ' body_false_response.' % statement_type)
+                   ' body_false_response.' % statement_type, mutant)
         if self.equal_with_limit(body_true_response,
                                  body_false_response,
                                  compare_diff):
@@ -162,14 +192,14 @@ def _find_bsql(self, mutant, statement_tuple, statement_type):
             # HTTP response body controlled by it is so small that the equal
             # ratio is not catching it.
             #
-            self.debug('Setting compare_diff to True')
+            self.debug('Setting compare_diff to True', mutant)
             compare_diff = True
 
         mutant.set_token_value(self.SYNTAX_ERROR)
         syntax_error_response, body_syntax_error_response = send_clean(mutant)
 
         self.debug('[%s] Comparing body_true_response and'
-                   ' body_syntax_error_response.' % statement_type)
+                   ' body_syntax_error_response.' % statement_type, mutant)
         if self.equal_with_limit(body_true_response,
                                  body_syntax_error_response,
                                  compare_diff):
@@ -182,7 +212,7 @@ def _find_bsql(self, mutant, statement_tuple, statement_type):
 
         # If they are equal then we have a search engine
         self.debug('[%s] Comparing body_true_response and'
-                   ' body_search_response.' % statement_type)
+                   ' body_search_response.' % statement_type, mutant)
         if self.equal_with_limit(body_true_response,
                                  body_search_response,
                                  compare_diff):
@@ -200,14 +230,14 @@ def _find_bsql(self, mutant, statement_tuple, statement_type):
         second_false_response, body_second_false_response = send_clean(mutant)
 
         self.debug('[%s] Comparing body_second_true_response and'
-                   ' body_true_response.' % statement_type)
+                   ' body_true_response.' % statement_type, mutant)
         if not self.equal_with_limit(body_second_true_response,
                                      body_true_response,
                                      compare_diff):
             return None
 
         self.debug('[%s] Comparing body_second_false_response and'
-                   ' body_false_response.' % statement_type)
+                   ' body_false_response.' % statement_type, mutant)
         if self.equal_with_limit(body_second_false_response,
                                  body_false_response,
                                  compare_diff):
@@ -235,8 +265,14 @@ def _find_bsql(self, mutant, statement_tuple, statement_type):
 
         return None
 
-    def debug(self, msg):
-        om.out.debug('[blind_sqli_debug] ' + msg)
+    def debug(self, msg, mutant=None):
+        if mutant is None:
+            log_line = '[blind_sqli_debug] %s' % msg
+        else:
+            args = (id(mutant), mutant.get_token_name(), msg)
+            log_line = '[blind_sqli_debug] [id: %s] [param: %s] %s' % args
+
+        om.out.debug(log_line)
 
     def equal_with_limit(self, body1, body2, compare_diff=False):
         """
@@ -248,7 +284,7 @@ def equal_with_limit(self, body1, body2, compare_diff=False):
         cmp_res = relative_distance_boolean(body1, body2, self._eq_limit)
 
         args = (self._eq_limit, cmp_res)
-        self.debug('Strings are similar enough with limit %s? %s' % args)
+        self.debug('Strings are similar enough with limit %s? %s' % args, None)
 
         return cmp_res
 

w3af/core/data/db/clean_dc.py

@@ -46,7 +46,7 @@ def clean_data_container(data_container):
         else:
             _type = 'string'
 
-        result.append('%s=%s' % (key, _type))
+        result.append('%s=%s' % (key.encode(DEFAULT_ENCODING), _type))
 
     return '&'.join(result)
 
@@ -102,7 +102,7 @@ def clean_path_filename(url):
     :param url: The URL instance
     :return: A clean URL string
     """
-    filename = url.get_file_name()
+    filename = url.get_file_name().encode(DEFAULT_ENCODING)
     path = url.get_path_without_file().encode(DEFAULT_ENCODING)
 
     if filename:

w3af/core/data/db/disk_list.py

@@ -257,4 +257,5 @@ def __len__(self):
     def __unicode__(self):
         return u'<DiskList [%s]>' % ', '.join([unicode(i) for i in self])
 
-    __str__ = __unicode__
+    __str__ = __unicode__
+

w3af/core/data/db/tests/test_variant_db.py

@@ -1,3 +1,4 @@
+# coding: utf8
 """
 test_variant_db.py
 
@@ -319,3 +320,27 @@ def test_db_same_with_qs(self):
 
         self.assertTrue(self.vdb.append(fr(url)))
         self.assertFalse(self.vdb.append(fr(url)))
+
+    def test_encoding_issues_se(self):
+        u = u'http://w3af.org/vård.png'
+        s = clean_fuzzable_request(fr(URL(u)))
+        e = u'(GET)-http://w3af.org/file-5692fef3f5dcd97.png'
+        self.assertEqual(s, e)
+
+    def test_encoding_issues_se_with_qs(self):
+        u = u'http://w3af.org/vård.png?id=1'
+        s = clean_fuzzable_request(fr(URL(u)))
+        e = '(GET)-http://w3af.org/vård.png?id=number'
+        self.assertEqual(s, e)
+
+    def test_encoding_issues_se_filename(self):
+        u = u'http://w3af.org/x.vård'
+        s = clean_fuzzable_request(fr(URL(u)))
+        e = '(GET)-http://w3af.org/file-5692fef3f5dcd97.vård'
+        self.assertEqual(s, e)
+
+    def test_encoding_issues_se_path(self):
+        u = u'http://w3af.org/vård/xyz.html'
+        s = clean_fuzzable_request(fr(URL(u)))
+        e = '(GET)-http://w3af.org/vård/file-5692fef3f5dcd97.html'
+        self.assertEqual(s, e)