Minor performance improvements

w3af/core/data/db/cached_disk_dict.py

@@ -19,6 +19,8 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 """
+from collections import Counter
+
 from w3af.core.data.db.disk_dict import DiskDict
 from w3af.core.data.fuzzer.utils import rand_alpha
 
@@ -43,7 +45,7 @@ def __init__(self, max_in_memory=50, table_prefix=None):
         self._max_in_memory = max_in_memory
         self._disk_dict = DiskDict(table_prefix=table_prefix)
         self._in_memory = dict()
-        self._access_count = dict()
+        self._access_count = Counter()
 
     def cleanup(self):
         self._disk_dict.cleanup()
@@ -89,22 +91,10 @@ def _get_keys_for_memory(self):
 
                 Then the method will generate [1, 2].
         """
-        items = self._access_count.items()
-        items.sort(sort_by_value)
-
-        iterator = min(self._max_in_memory, len(items))
-
-        result = []
-
-        for i in xrange(iterator):
-            result.append(items[i][0])
-
-        return result
+        return [k for k, v in self._access_count.most_common(self._max_in_memory)]
 
     def _increase_access_count(self, key):
-        access_count = self._access_count.get(key, 0)
-        access_count += 1
-        self._access_count[key] = access_count
+        self._access_count.update([key])
 
         keys_for_memory = self._get_keys_for_memory()
 

w3af/core/data/db/variant_db.py

@@ -132,15 +132,20 @@ def append(self, fuzzable_request):
                  False if NO more variants are required for this fuzzable
                  request.
         """
+        request_hash = fuzzable_request.get_request_hash(self.HASH_IGNORE_HEADERS)
+
         with self._db_lock:
-            if self._seen_exactly_the_same(fuzzable_request):
+            if self._seen_exactly_the_same(fuzzable_request, request_hash):
+                self._log_return_false(fuzzable_request, 'seen_exactly_the_same')
                 return False
 
             if self._has_form(fuzzable_request):
                 if not self._need_more_variants_for_form(fuzzable_request):
+                    self._log_return_false(fuzzable_request, 'need_more_variants_for_form')
                     return False
 
             if not self._need_more_variants_for_uri(fuzzable_request):
+                self._log_return_false(fuzzable_request, 'need_more_variants_for_uri')
                 return False
 
             # Yes, please give me more variants of fuzzable_request
@@ -183,11 +188,10 @@ def _need_more_variants_for_uri(self, fuzzable_request):
         self._variants[clean_dict_key] = count + 1
         return True
 
-    def _seen_exactly_the_same(self, fuzzable_request):
+    def _seen_exactly_the_same(self, fuzzable_request, request_hash):
         #
         # Is the fuzzable request already known to us? (exactly the same)
         #
-        request_hash = fuzzable_request.get_request_hash(self.HASH_IGNORE_HEADERS)
         if request_hash in self._variants_eq:
             return True